git-p4.pyon commit git-p4: improve path encoding verbose output (00a9403)
   1#!/usr/bin/env python
   2#
   3# git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4#
   5# Author: Simon Hausmann <simon@lst.de>
   6# Copyright: 2007 Simon Hausmann <simon@lst.de>
   7#            2007 Trolltech ASA
   8# License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9#
  10import sys
  11if sys.hexversion < 0x02040000:
  12    # The limiter is the subprocess module
  13    sys.stderr.write("git-p4: requires Python 2.4 or later.\n")
  14    sys.exit(1)
  15import os
  16import optparse
  17import marshal
  18import subprocess
  19import tempfile
  20import time
  21import platform
  22import re
  23import shutil
  24import stat
  25
  26try:
  27    from subprocess import CalledProcessError
  28except ImportError:
  29    # from python2.7:subprocess.py
  30    # Exception classes used by this module.
  31    class CalledProcessError(Exception):
  32        """This exception is raised when a process run by check_call() returns
  33        a non-zero exit status.  The exit status will be stored in the
  34        returncode attribute."""
  35        def __init__(self, returncode, cmd):
  36            self.returncode = returncode
  37            self.cmd = cmd
  38        def __str__(self):
  39            return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode)
  40
  41verbose = False
  42
  43# Only labels/tags matching this will be imported/exported
  44defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$'
  45
  46# Grab changes in blocks of this many revisions, unless otherwise requested
  47defaultBlockSize = 512
  48
  49def p4_build_cmd(cmd):
  50    """Build a suitable p4 command line.
  51
  52    This consolidates building and returning a p4 command line into one
  53    location. It means that hooking into the environment, or other configuration
  54    can be done more easily.
  55    """
  56    real_cmd = ["p4"]
  57
  58    user = gitConfig("git-p4.user")
  59    if len(user) > 0:
  60        real_cmd += ["-u",user]
  61
  62    password = gitConfig("git-p4.password")
  63    if len(password) > 0:
  64        real_cmd += ["-P", password]
  65
  66    port = gitConfig("git-p4.port")
  67    if len(port) > 0:
  68        real_cmd += ["-p", port]
  69
  70    host = gitConfig("git-p4.host")
  71    if len(host) > 0:
  72        real_cmd += ["-H", host]
  73
  74    client = gitConfig("git-p4.client")
  75    if len(client) > 0:
  76        real_cmd += ["-c", client]
  77
  78
  79    if isinstance(cmd,basestring):
  80        real_cmd = ' '.join(real_cmd) + ' ' + cmd
  81    else:
  82        real_cmd += cmd
  83    return real_cmd
  84
  85def chdir(path, is_client_path=False):
  86    """Do chdir to the given path, and set the PWD environment
  87       variable for use by P4.  It does not look at getcwd() output.
  88       Since we're not using the shell, it is necessary to set the
  89       PWD environment variable explicitly.
  90
  91       Normally, expand the path to force it to be absolute.  This
  92       addresses the use of relative path names inside P4 settings,
  93       e.g. P4CONFIG=.p4config.  P4 does not simply open the filename
  94       as given; it looks for .p4config using PWD.
  95
  96       If is_client_path, the path was handed to us directly by p4,
  97       and may be a symbolic link.  Do not call os.getcwd() in this
  98       case, because it will cause p4 to think that PWD is not inside
  99       the client path.
 100       """
 101
 102    os.chdir(path)
 103    if not is_client_path:
 104        path = os.getcwd()
 105    os.environ['PWD'] = path
 106
 107def die(msg):
 108    if verbose:
 109        raise Exception(msg)
 110    else:
 111        sys.stderr.write(msg + "\n")
 112        sys.exit(1)
 113
 114def write_pipe(c, stdin):
 115    if verbose:
 116        sys.stderr.write('Writing pipe: %s\n' % str(c))
 117
 118    expand = isinstance(c,basestring)
 119    p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
 120    pipe = p.stdin
 121    val = pipe.write(stdin)
 122    pipe.close()
 123    if p.wait():
 124        die('Command failed: %s' % str(c))
 125
 126    return val
 127
 128def p4_write_pipe(c, stdin):
 129    real_cmd = p4_build_cmd(c)
 130    return write_pipe(real_cmd, stdin)
 131
 132def read_pipe(c, ignore_error=False):
 133    if verbose:
 134        sys.stderr.write('Reading pipe: %s\n' % str(c))
 135
 136    expand = isinstance(c,basestring)
 137    p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
 138    pipe = p.stdout
 139    val = pipe.read()
 140    if p.wait() and not ignore_error:
 141        die('Command failed: %s' % str(c))
 142
 143    return val
 144
 145def p4_read_pipe(c, ignore_error=False):
 146    real_cmd = p4_build_cmd(c)
 147    return read_pipe(real_cmd, ignore_error)
 148
 149def read_pipe_lines(c):
 150    if verbose:
 151        sys.stderr.write('Reading pipe: %s\n' % str(c))
 152
 153    expand = isinstance(c, basestring)
 154    p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
 155    pipe = p.stdout
 156    val = pipe.readlines()
 157    if pipe.close() or p.wait():
 158        die('Command failed: %s' % str(c))
 159
 160    return val
 161
 162def p4_read_pipe_lines(c):
 163    """Specifically invoke p4 on the command supplied. """
 164    real_cmd = p4_build_cmd(c)
 165    return read_pipe_lines(real_cmd)
 166
 167def p4_has_command(cmd):
 168    """Ask p4 for help on this command.  If it returns an error, the
 169       command does not exist in this version of p4."""
 170    real_cmd = p4_build_cmd(["help", cmd])
 171    p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE,
 172                                   stderr=subprocess.PIPE)
 173    p.communicate()
 174    return p.returncode == 0
 175
 176def p4_has_move_command():
 177    """See if the move command exists, that it supports -k, and that
 178       it has not been administratively disabled.  The arguments
 179       must be correct, but the filenames do not have to exist.  Use
 180       ones with wildcards so even if they exist, it will fail."""
 181
 182    if not p4_has_command("move"):
 183        return False
 184    cmd = p4_build_cmd(["move", "-k", "@from", "@to"])
 185    p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 186    (out, err) = p.communicate()
 187    # return code will be 1 in either case
 188    if err.find("Invalid option") >= 0:
 189        return False
 190    if err.find("disabled") >= 0:
 191        return False
 192    # assume it failed because @... was invalid changelist
 193    return True
 194
 195def system(cmd):
 196    expand = isinstance(cmd,basestring)
 197    if verbose:
 198        sys.stderr.write("executing %s\n" % str(cmd))
 199    retcode = subprocess.call(cmd, shell=expand)
 200    if retcode:
 201        raise CalledProcessError(retcode, cmd)
 202
 203def p4_system(cmd):
 204    """Specifically invoke p4 as the system command. """
 205    real_cmd = p4_build_cmd(cmd)
 206    expand = isinstance(real_cmd, basestring)
 207    retcode = subprocess.call(real_cmd, shell=expand)
 208    if retcode:
 209        raise CalledProcessError(retcode, real_cmd)
 210
 211_p4_version_string = None
 212def p4_version_string():
 213    """Read the version string, showing just the last line, which
 214       hopefully is the interesting version bit.
 215
 216       $ p4 -V
 217       Perforce - The Fast Software Configuration Management System.
 218       Copyright 1995-2011 Perforce Software.  All rights reserved.
 219       Rev. P4/NTX86/2011.1/393975 (2011/12/16).
 220    """
 221    global _p4_version_string
 222    if not _p4_version_string:
 223        a = p4_read_pipe_lines(["-V"])
 224        _p4_version_string = a[-1].rstrip()
 225    return _p4_version_string
 226
 227def p4_integrate(src, dest):
 228    p4_system(["integrate", "-Dt", wildcard_encode(src), wildcard_encode(dest)])
 229
 230def p4_sync(f, *options):
 231    p4_system(["sync"] + list(options) + [wildcard_encode(f)])
 232
 233def p4_add(f):
 234    # forcibly add file names with wildcards
 235    if wildcard_present(f):
 236        p4_system(["add", "-f", f])
 237    else:
 238        p4_system(["add", f])
 239
 240def p4_delete(f):
 241    p4_system(["delete", wildcard_encode(f)])
 242
 243def p4_edit(f):
 244    p4_system(["edit", wildcard_encode(f)])
 245
 246def p4_revert(f):
 247    p4_system(["revert", wildcard_encode(f)])
 248
 249def p4_reopen(type, f):
 250    p4_system(["reopen", "-t", type, wildcard_encode(f)])
 251
 252def p4_move(src, dest):
 253    p4_system(["move", "-k", wildcard_encode(src), wildcard_encode(dest)])
 254
 255def p4_last_change():
 256    results = p4CmdList(["changes", "-m", "1"])
 257    return int(results[0]['change'])
 258
 259def p4_describe(change):
 260    """Make sure it returns a valid result by checking for
 261       the presence of field "time".  Return a dict of the
 262       results."""
 263
 264    ds = p4CmdList(["describe", "-s", str(change)])
 265    if len(ds) != 1:
 266        die("p4 describe -s %d did not return 1 result: %s" % (change, str(ds)))
 267
 268    d = ds[0]
 269
 270    if "p4ExitCode" in d:
 271        die("p4 describe -s %d exited with %d: %s" % (change, d["p4ExitCode"],
 272                                                      str(d)))
 273    if "code" in d:
 274        if d["code"] == "error":
 275            die("p4 describe -s %d returned error code: %s" % (change, str(d)))
 276
 277    if "time" not in d:
 278        die("p4 describe -s %d returned no \"time\": %s" % (change, str(d)))
 279
 280    return d
 281
 282#
 283# Canonicalize the p4 type and return a tuple of the
 284# base type, plus any modifiers.  See "p4 help filetypes"
 285# for a list and explanation.
 286#
 287def split_p4_type(p4type):
 288
 289    p4_filetypes_historical = {
 290        "ctempobj": "binary+Sw",
 291        "ctext": "text+C",
 292        "cxtext": "text+Cx",
 293        "ktext": "text+k",
 294        "kxtext": "text+kx",
 295        "ltext": "text+F",
 296        "tempobj": "binary+FSw",
 297        "ubinary": "binary+F",
 298        "uresource": "resource+F",
 299        "uxbinary": "binary+Fx",
 300        "xbinary": "binary+x",
 301        "xltext": "text+Fx",
 302        "xtempobj": "binary+Swx",
 303        "xtext": "text+x",
 304        "xunicode": "unicode+x",
 305        "xutf16": "utf16+x",
 306    }
 307    if p4type in p4_filetypes_historical:
 308        p4type = p4_filetypes_historical[p4type]
 309    mods = ""
 310    s = p4type.split("+")
 311    base = s[0]
 312    mods = ""
 313    if len(s) > 1:
 314        mods = s[1]
 315    return (base, mods)
 316
 317#
 318# return the raw p4 type of a file (text, text+ko, etc)
 319#
 320def p4_type(f):
 321    results = p4CmdList(["fstat", "-T", "headType", wildcard_encode(f)])
 322    return results[0]['headType']
 323
 324#
 325# Given a type base and modifier, return a regexp matching
 326# the keywords that can be expanded in the file
 327#
 328def p4_keywords_regexp_for_type(base, type_mods):
 329    if base in ("text", "unicode", "binary"):
 330        kwords = None
 331        if "ko" in type_mods:
 332            kwords = 'Id|Header'
 333        elif "k" in type_mods:
 334            kwords = 'Id|Header|Author|Date|DateTime|Change|File|Revision'
 335        else:
 336            return None
 337        pattern = r"""
 338            \$              # Starts with a dollar, followed by...
 339            (%s)            # one of the keywords, followed by...
 340            (:[^$\n]+)?     # possibly an old expansion, followed by...
 341            \$              # another dollar
 342            """ % kwords
 343        return pattern
 344    else:
 345        return None
 346
 347#
 348# Given a file, return a regexp matching the possible
 349# RCS keywords that will be expanded, or None for files
 350# with kw expansion turned off.
 351#
 352def p4_keywords_regexp_for_file(file):
 353    if not os.path.exists(file):
 354        return None
 355    else:
 356        (type_base, type_mods) = split_p4_type(p4_type(file))
 357        return p4_keywords_regexp_for_type(type_base, type_mods)
 358
 359def setP4ExecBit(file, mode):
 360    # Reopens an already open file and changes the execute bit to match
 361    # the execute bit setting in the passed in mode.
 362
 363    p4Type = "+x"
 364
 365    if not isModeExec(mode):
 366        p4Type = getP4OpenedType(file)
 367        p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
 368        p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
 369        if p4Type[-1] == "+":
 370            p4Type = p4Type[0:-1]
 371
 372    p4_reopen(p4Type, file)
 373
 374def getP4OpenedType(file):
 375    # Returns the perforce file type for the given file.
 376
 377    result = p4_read_pipe(["opened", wildcard_encode(file)])
 378    match = re.match(".*\((.+)\)( \*exclusive\*)?\r?$", result)
 379    if match:
 380        return match.group(1)
 381    else:
 382        die("Could not determine file type for %s (result: '%s')" % (file, result))
 383
 384# Return the set of all p4 labels
 385def getP4Labels(depotPaths):
 386    labels = set()
 387    if isinstance(depotPaths,basestring):
 388        depotPaths = [depotPaths]
 389
 390    for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]):
 391        label = l['label']
 392        labels.add(label)
 393
 394    return labels
 395
 396# Return the set of all git tags
 397def getGitTags():
 398    gitTags = set()
 399    for line in read_pipe_lines(["git", "tag"]):
 400        tag = line.strip()
 401        gitTags.add(tag)
 402    return gitTags
 403
 404def diffTreePattern():
 405    # This is a simple generator for the diff tree regex pattern. This could be
 406    # a class variable if this and parseDiffTreeEntry were a part of a class.
 407    pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
 408    while True:
 409        yield pattern
 410
 411def parseDiffTreeEntry(entry):
 412    """Parses a single diff tree entry into its component elements.
 413
 414    See git-diff-tree(1) manpage for details about the format of the diff
 415    output. This method returns a dictionary with the following elements:
 416
 417    src_mode - The mode of the source file
 418    dst_mode - The mode of the destination file
 419    src_sha1 - The sha1 for the source file
 420    dst_sha1 - The sha1 fr the destination file
 421    status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
 422    status_score - The score for the status (applicable for 'C' and 'R'
 423                   statuses). This is None if there is no score.
 424    src - The path for the source file.
 425    dst - The path for the destination file. This is only present for
 426          copy or renames. If it is not present, this is None.
 427
 428    If the pattern is not matched, None is returned."""
 429
 430    match = diffTreePattern().next().match(entry)
 431    if match:
 432        return {
 433            'src_mode': match.group(1),
 434            'dst_mode': match.group(2),
 435            'src_sha1': match.group(3),
 436            'dst_sha1': match.group(4),
 437            'status': match.group(5),
 438            'status_score': match.group(6),
 439            'src': match.group(7),
 440            'dst': match.group(10)
 441        }
 442    return None
 443
 444def isModeExec(mode):
 445    # Returns True if the given git mode represents an executable file,
 446    # otherwise False.
 447    return mode[-3:] == "755"
 448
 449def isModeExecChanged(src_mode, dst_mode):
 450    return isModeExec(src_mode) != isModeExec(dst_mode)
 451
 452def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
 453
 454    if isinstance(cmd,basestring):
 455        cmd = "-G " + cmd
 456        expand = True
 457    else:
 458        cmd = ["-G"] + cmd
 459        expand = False
 460
 461    cmd = p4_build_cmd(cmd)
 462    if verbose:
 463        sys.stderr.write("Opening pipe: %s\n" % str(cmd))
 464
 465    # Use a temporary file to avoid deadlocks without
 466    # subprocess.communicate(), which would put another copy
 467    # of stdout into memory.
 468    stdin_file = None
 469    if stdin is not None:
 470        stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
 471        if isinstance(stdin,basestring):
 472            stdin_file.write(stdin)
 473        else:
 474            for i in stdin:
 475                stdin_file.write(i + '\n')
 476        stdin_file.flush()
 477        stdin_file.seek(0)
 478
 479    p4 = subprocess.Popen(cmd,
 480                          shell=expand,
 481                          stdin=stdin_file,
 482                          stdout=subprocess.PIPE)
 483
 484    result = []
 485    try:
 486        while True:
 487            entry = marshal.load(p4.stdout)
 488            if cb is not None:
 489                cb(entry)
 490            else:
 491                result.append(entry)
 492    except EOFError:
 493        pass
 494    exitCode = p4.wait()
 495    if exitCode != 0:
 496        entry = {}
 497        entry["p4ExitCode"] = exitCode
 498        result.append(entry)
 499
 500    return result
 501
 502def p4Cmd(cmd):
 503    list = p4CmdList(cmd)
 504    result = {}
 505    for entry in list:
 506        result.update(entry)
 507    return result;
 508
 509def p4Where(depotPath):
 510    if not depotPath.endswith("/"):
 511        depotPath += "/"
 512    depotPathLong = depotPath + "..."
 513    outputList = p4CmdList(["where", depotPathLong])
 514    output = None
 515    for entry in outputList:
 516        if "depotFile" in entry:
 517            # Search for the base client side depot path, as long as it starts with the branch's P4 path.
 518            # The base path always ends with "/...".
 519            if entry["depotFile"].find(depotPath) == 0 and entry["depotFile"][-4:] == "/...":
 520                output = entry
 521                break
 522        elif "data" in entry:
 523            data = entry.get("data")
 524            space = data.find(" ")
 525            if data[:space] == depotPath:
 526                output = entry
 527                break
 528    if output == None:
 529        return ""
 530    if output["code"] == "error":
 531        return ""
 532    clientPath = ""
 533    if "path" in output:
 534        clientPath = output.get("path")
 535    elif "data" in output:
 536        data = output.get("data")
 537        lastSpace = data.rfind(" ")
 538        clientPath = data[lastSpace + 1:]
 539
 540    if clientPath.endswith("..."):
 541        clientPath = clientPath[:-3]
 542    return clientPath
 543
 544def currentGitBranch():
 545    return read_pipe("git name-rev HEAD").split(" ")[1].strip()
 546
 547def isValidGitDir(path):
 548    if (os.path.exists(path + "/HEAD")
 549        and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
 550        return True;
 551    return False
 552
 553def parseRevision(ref):
 554    return read_pipe("git rev-parse %s" % ref).strip()
 555
 556def branchExists(ref):
 557    rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
 558                     ignore_error=True)
 559    return len(rev) > 0
 560
 561def extractLogMessageFromGitCommit(commit):
 562    logMessage = ""
 563
 564    ## fixme: title is first line of commit, not 1st paragraph.
 565    foundTitle = False
 566    for log in read_pipe_lines("git cat-file commit %s" % commit):
 567       if not foundTitle:
 568           if len(log) == 1:
 569               foundTitle = True
 570           continue
 571
 572       logMessage += log
 573    return logMessage
 574
 575def extractSettingsGitLog(log):
 576    values = {}
 577    for line in log.split("\n"):
 578        line = line.strip()
 579        m = re.search (r"^ *\[git-p4: (.*)\]$", line)
 580        if not m:
 581            continue
 582
 583        assignments = m.group(1).split (':')
 584        for a in assignments:
 585            vals = a.split ('=')
 586            key = vals[0].strip()
 587            val = ('='.join (vals[1:])).strip()
 588            if val.endswith ('\"') and val.startswith('"'):
 589                val = val[1:-1]
 590
 591            values[key] = val
 592
 593    paths = values.get("depot-paths")
 594    if not paths:
 595        paths = values.get("depot-path")
 596    if paths:
 597        values['depot-paths'] = paths.split(',')
 598    return values
 599
 600def gitBranchExists(branch):
 601    proc = subprocess.Popen(["git", "rev-parse", branch],
 602                            stderr=subprocess.PIPE, stdout=subprocess.PIPE);
 603    return proc.wait() == 0;
 604
 605_gitConfig = {}
 606
 607def gitConfig(key):
 608    if not _gitConfig.has_key(key):
 609        cmd = [ "git", "config", key ]
 610        s = read_pipe(cmd, ignore_error=True)
 611        _gitConfig[key] = s.strip()
 612    return _gitConfig[key]
 613
 614def gitConfigBool(key):
 615    """Return a bool, using git config --bool.  It is True only if the
 616       variable is set to true, and False if set to false or not present
 617       in the config."""
 618
 619    if not _gitConfig.has_key(key):
 620        cmd = [ "git", "config", "--bool", key ]
 621        s = read_pipe(cmd, ignore_error=True)
 622        v = s.strip()
 623        _gitConfig[key] = v == "true"
 624    return _gitConfig[key]
 625
 626def gitConfigList(key):
 627    if not _gitConfig.has_key(key):
 628        s = read_pipe(["git", "config", "--get-all", key], ignore_error=True)
 629        _gitConfig[key] = s.strip().split(os.linesep)
 630    return _gitConfig[key]
 631
 632def p4BranchesInGit(branchesAreInRemotes=True):
 633    """Find all the branches whose names start with "p4/", looking
 634       in remotes or heads as specified by the argument.  Return
 635       a dictionary of { branch: revision } for each one found.
 636       The branch names are the short names, without any
 637       "p4/" prefix."""
 638
 639    branches = {}
 640
 641    cmdline = "git rev-parse --symbolic "
 642    if branchesAreInRemotes:
 643        cmdline += "--remotes"
 644    else:
 645        cmdline += "--branches"
 646
 647    for line in read_pipe_lines(cmdline):
 648        line = line.strip()
 649
 650        # only import to p4/
 651        if not line.startswith('p4/'):
 652            continue
 653        # special symbolic ref to p4/master
 654        if line == "p4/HEAD":
 655            continue
 656
 657        # strip off p4/ prefix
 658        branch = line[len("p4/"):]
 659
 660        branches[branch] = parseRevision(line)
 661
 662    return branches
 663
 664def branch_exists(branch):
 665    """Make sure that the given ref name really exists."""
 666
 667    cmd = [ "git", "rev-parse", "--symbolic", "--verify", branch ]
 668    p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 669    out, _ = p.communicate()
 670    if p.returncode:
 671        return False
 672    # expect exactly one line of output: the branch name
 673    return out.rstrip() == branch
 674
 675def findUpstreamBranchPoint(head = "HEAD"):
 676    branches = p4BranchesInGit()
 677    # map from depot-path to branch name
 678    branchByDepotPath = {}
 679    for branch in branches.keys():
 680        tip = branches[branch]
 681        log = extractLogMessageFromGitCommit(tip)
 682        settings = extractSettingsGitLog(log)
 683        if settings.has_key("depot-paths"):
 684            paths = ",".join(settings["depot-paths"])
 685            branchByDepotPath[paths] = "remotes/p4/" + branch
 686
 687    settings = None
 688    parent = 0
 689    while parent < 65535:
 690        commit = head + "~%s" % parent
 691        log = extractLogMessageFromGitCommit(commit)
 692        settings = extractSettingsGitLog(log)
 693        if settings.has_key("depot-paths"):
 694            paths = ",".join(settings["depot-paths"])
 695            if branchByDepotPath.has_key(paths):
 696                return [branchByDepotPath[paths], settings]
 697
 698        parent = parent + 1
 699
 700    return ["", settings]
 701
 702def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
 703    if not silent:
 704        print ("Creating/updating branch(es) in %s based on origin branch(es)"
 705               % localRefPrefix)
 706
 707    originPrefix = "origin/p4/"
 708
 709    for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
 710        line = line.strip()
 711        if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
 712            continue
 713
 714        headName = line[len(originPrefix):]
 715        remoteHead = localRefPrefix + headName
 716        originHead = line
 717
 718        original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
 719        if (not original.has_key('depot-paths')
 720            or not original.has_key('change')):
 721            continue
 722
 723        update = False
 724        if not gitBranchExists(remoteHead):
 725            if verbose:
 726                print "creating %s" % remoteHead
 727            update = True
 728        else:
 729            settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
 730            if settings.has_key('change') > 0:
 731                if settings['depot-paths'] == original['depot-paths']:
 732                    originP4Change = int(original['change'])
 733                    p4Change = int(settings['change'])
 734                    if originP4Change > p4Change:
 735                        print ("%s (%s) is newer than %s (%s). "
 736                               "Updating p4 branch from origin."
 737                               % (originHead, originP4Change,
 738                                  remoteHead, p4Change))
 739                        update = True
 740                else:
 741                    print ("Ignoring: %s was imported from %s while "
 742                           "%s was imported from %s"
 743                           % (originHead, ','.join(original['depot-paths']),
 744                              remoteHead, ','.join(settings['depot-paths'])))
 745
 746        if update:
 747            system("git update-ref %s %s" % (remoteHead, originHead))
 748
 749def originP4BranchesExist():
 750        return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
 751
 752
 753def p4ParseNumericChangeRange(parts):
 754    changeStart = int(parts[0][1:])
 755    if parts[1] == '#head':
 756        changeEnd = p4_last_change()
 757    else:
 758        changeEnd = int(parts[1])
 759
 760    return (changeStart, changeEnd)
 761
 762def chooseBlockSize(blockSize):
 763    if blockSize:
 764        return blockSize
 765    else:
 766        return defaultBlockSize
 767
 768def p4ChangesForPaths(depotPaths, changeRange, requestedBlockSize):
 769    assert depotPaths
 770
 771    # Parse the change range into start and end. Try to find integer
 772    # revision ranges as these can be broken up into blocks to avoid
 773    # hitting server-side limits (maxrows, maxscanresults). But if
 774    # that doesn't work, fall back to using the raw revision specifier
 775    # strings, without using block mode.
 776
 777    if changeRange is None or changeRange == '':
 778        changeStart = 1
 779        changeEnd = p4_last_change()
 780        block_size = chooseBlockSize(requestedBlockSize)
 781    else:
 782        parts = changeRange.split(',')
 783        assert len(parts) == 2
 784        try:
 785            (changeStart, changeEnd) = p4ParseNumericChangeRange(parts)
 786            block_size = chooseBlockSize(requestedBlockSize)
 787        except:
 788            changeStart = parts[0][1:]
 789            changeEnd = parts[1]
 790            if requestedBlockSize:
 791                die("cannot use --changes-block-size with non-numeric revisions")
 792            block_size = None
 793
 794    # Accumulate change numbers in a dictionary to avoid duplicates
 795    changes = {}
 796
 797    for p in depotPaths:
 798        # Retrieve changes a block at a time, to prevent running
 799        # into a MaxResults/MaxScanRows error from the server.
 800
 801        while True:
 802            cmd = ['changes']
 803
 804            if block_size:
 805                end = min(changeEnd, changeStart + block_size)
 806                revisionRange = "%d,%d" % (changeStart, end)
 807            else:
 808                revisionRange = "%s,%s" % (changeStart, changeEnd)
 809
 810            cmd += ["%s...@%s" % (p, revisionRange)]
 811
 812            for line in p4_read_pipe_lines(cmd):
 813                changeNum = int(line.split(" ")[1])
 814                changes[changeNum] = True
 815
 816            if not block_size:
 817                break
 818
 819            if end >= changeEnd:
 820                break
 821
 822            changeStart = end + 1
 823
 824    changelist = changes.keys()
 825    changelist.sort()
 826    return changelist
 827
 828def p4PathStartsWith(path, prefix):
 829    # This method tries to remedy a potential mixed-case issue:
 830    #
 831    # If UserA adds  //depot/DirA/file1
 832    # and UserB adds //depot/dira/file2
 833    #
 834    # we may or may not have a problem. If you have core.ignorecase=true,
 835    # we treat DirA and dira as the same directory
 836    if gitConfigBool("core.ignorecase"):
 837        return path.lower().startswith(prefix.lower())
 838    return path.startswith(prefix)
 839
 840def getClientSpec():
 841    """Look at the p4 client spec, create a View() object that contains
 842       all the mappings, and return it."""
 843
 844    specList = p4CmdList("client -o")
 845    if len(specList) != 1:
 846        die('Output from "client -o" is %d lines, expecting 1' %
 847            len(specList))
 848
 849    # dictionary of all client parameters
 850    entry = specList[0]
 851
 852    # the //client/ name
 853    client_name = entry["Client"]
 854
 855    # just the keys that start with "View"
 856    view_keys = [ k for k in entry.keys() if k.startswith("View") ]
 857
 858    # hold this new View
 859    view = View(client_name)
 860
 861    # append the lines, in order, to the view
 862    for view_num in range(len(view_keys)):
 863        k = "View%d" % view_num
 864        if k not in view_keys:
 865            die("Expected view key %s missing" % k)
 866        view.append(entry[k])
 867
 868    return view
 869
 870def getClientRoot():
 871    """Grab the client directory."""
 872
 873    output = p4CmdList("client -o")
 874    if len(output) != 1:
 875        die('Output from "client -o" is %d lines, expecting 1' % len(output))
 876
 877    entry = output[0]
 878    if "Root" not in entry:
 879        die('Client has no "Root"')
 880
 881    return entry["Root"]
 882
 883#
 884# P4 wildcards are not allowed in filenames.  P4 complains
 885# if you simply add them, but you can force it with "-f", in
 886# which case it translates them into %xx encoding internally.
 887#
 888def wildcard_decode(path):
 889    # Search for and fix just these four characters.  Do % last so
 890    # that fixing it does not inadvertently create new %-escapes.
 891    # Cannot have * in a filename in windows; untested as to
 892    # what p4 would do in such a case.
 893    if not platform.system() == "Windows":
 894        path = path.replace("%2A", "*")
 895    path = path.replace("%23", "#") \
 896               .replace("%40", "@") \
 897               .replace("%25", "%")
 898    return path
 899
 900def wildcard_encode(path):
 901    # do % first to avoid double-encoding the %s introduced here
 902    path = path.replace("%", "%25") \
 903               .replace("*", "%2A") \
 904               .replace("#", "%23") \
 905               .replace("@", "%40")
 906    return path
 907
 908def wildcard_present(path):
 909    m = re.search("[*#@%]", path)
 910    return m is not None
 911
 912class Command:
 913    def __init__(self):
 914        self.usage = "usage: %prog [options]"
 915        self.needsGit = True
 916        self.verbose = False
 917
 918class P4UserMap:
 919    def __init__(self):
 920        self.userMapFromPerforceServer = False
 921        self.myP4UserId = None
 922
 923    def p4UserId(self):
 924        if self.myP4UserId:
 925            return self.myP4UserId
 926
 927        results = p4CmdList("user -o")
 928        for r in results:
 929            if r.has_key('User'):
 930                self.myP4UserId = r['User']
 931                return r['User']
 932        die("Could not find your p4 user id")
 933
 934    def p4UserIsMe(self, p4User):
 935        # return True if the given p4 user is actually me
 936        me = self.p4UserId()
 937        if not p4User or p4User != me:
 938            return False
 939        else:
 940            return True
 941
 942    def getUserCacheFilename(self):
 943        home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
 944        return home + "/.gitp4-usercache.txt"
 945
 946    def getUserMapFromPerforceServer(self):
 947        if self.userMapFromPerforceServer:
 948            return
 949        self.users = {}
 950        self.emails = {}
 951
 952        for output in p4CmdList("users"):
 953            if not output.has_key("User"):
 954                continue
 955            self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
 956            self.emails[output["Email"]] = output["User"]
 957
 958
 959        s = ''
 960        for (key, val) in self.users.items():
 961            s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
 962
 963        open(self.getUserCacheFilename(), "wb").write(s)
 964        self.userMapFromPerforceServer = True
 965
 966    def loadUserMapFromCache(self):
 967        self.users = {}
 968        self.userMapFromPerforceServer = False
 969        try:
 970            cache = open(self.getUserCacheFilename(), "rb")
 971            lines = cache.readlines()
 972            cache.close()
 973            for line in lines:
 974                entry = line.strip().split("\t")
 975                self.users[entry[0]] = entry[1]
 976        except IOError:
 977            self.getUserMapFromPerforceServer()
 978
 979class P4Debug(Command):
 980    def __init__(self):
 981        Command.__init__(self)
 982        self.options = []
 983        self.description = "A tool to debug the output of p4 -G."
 984        self.needsGit = False
 985
 986    def run(self, args):
 987        j = 0
 988        for output in p4CmdList(args):
 989            print 'Element: %d' % j
 990            j += 1
 991            print output
 992        return True
 993
 994class P4RollBack(Command):
 995    def __init__(self):
 996        Command.__init__(self)
 997        self.options = [
 998            optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
 999        ]
1000        self.description = "A tool to debug the multi-branch import. Don't use :)"
1001        self.rollbackLocalBranches = False
1002
1003    def run(self, args):
1004        if len(args) != 1:
1005            return False
1006        maxChange = int(args[0])
1007
1008        if "p4ExitCode" in p4Cmd("changes -m 1"):
1009            die("Problems executing p4");
1010
1011        if self.rollbackLocalBranches:
1012            refPrefix = "refs/heads/"
1013            lines = read_pipe_lines("git rev-parse --symbolic --branches")
1014        else:
1015            refPrefix = "refs/remotes/"
1016            lines = read_pipe_lines("git rev-parse --symbolic --remotes")
1017
1018        for line in lines:
1019            if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
1020                line = line.strip()
1021                ref = refPrefix + line
1022                log = extractLogMessageFromGitCommit(ref)
1023                settings = extractSettingsGitLog(log)
1024
1025                depotPaths = settings['depot-paths']
1026                change = settings['change']
1027
1028                changed = False
1029
1030                if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
1031                                                           for p in depotPaths]))) == 0:
1032                    print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
1033                    system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
1034                    continue
1035
1036                while change and int(change) > maxChange:
1037                    changed = True
1038                    if self.verbose:
1039                        print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
1040                    system("git update-ref %s \"%s^\"" % (ref, ref))
1041                    log = extractLogMessageFromGitCommit(ref)
1042                    settings =  extractSettingsGitLog(log)
1043
1044
1045                    depotPaths = settings['depot-paths']
1046                    change = settings['change']
1047
1048                if changed:
1049                    print "%s rewound to %s" % (ref, change)
1050
1051        return True
1052
1053class P4Submit(Command, P4UserMap):
1054
1055    conflict_behavior_choices = ("ask", "skip", "quit")
1056
1057    def __init__(self):
1058        Command.__init__(self)
1059        P4UserMap.__init__(self)
1060        self.options = [
1061                optparse.make_option("--origin", dest="origin"),
1062                optparse.make_option("-M", dest="detectRenames", action="store_true"),
1063                # preserve the user, requires relevant p4 permissions
1064                optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
1065                optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),
1066                optparse.make_option("--dry-run", "-n", dest="dry_run", action="store_true"),
1067                optparse.make_option("--prepare-p4-only", dest="prepare_p4_only", action="store_true"),
1068                optparse.make_option("--conflict", dest="conflict_behavior",
1069                                     choices=self.conflict_behavior_choices),
1070                optparse.make_option("--branch", dest="branch"),
1071        ]
1072        self.description = "Submit changes from git to the perforce depot."
1073        self.usage += " [name of git branch to submit into perforce depot]"
1074        self.origin = ""
1075        self.detectRenames = False
1076        self.preserveUser = gitConfigBool("git-p4.preserveUser")
1077        self.dry_run = False
1078        self.prepare_p4_only = False
1079        self.conflict_behavior = None
1080        self.isWindows = (platform.system() == "Windows")
1081        self.exportLabels = False
1082        self.p4HasMoveCommand = p4_has_move_command()
1083        self.branch = None
1084
1085    def check(self):
1086        if len(p4CmdList("opened ...")) > 0:
1087            die("You have files opened with perforce! Close them before starting the sync.")
1088
1089    def separate_jobs_from_description(self, message):
1090        """Extract and return a possible Jobs field in the commit
1091           message.  It goes into a separate section in the p4 change
1092           specification.
1093
1094           A jobs line starts with "Jobs:" and looks like a new field
1095           in a form.  Values are white-space separated on the same
1096           line or on following lines that start with a tab.
1097
1098           This does not parse and extract the full git commit message
1099           like a p4 form.  It just sees the Jobs: line as a marker
1100           to pass everything from then on directly into the p4 form,
1101           but outside the description section.
1102
1103           Return a tuple (stripped log message, jobs string)."""
1104
1105        m = re.search(r'^Jobs:', message, re.MULTILINE)
1106        if m is None:
1107            return (message, None)
1108
1109        jobtext = message[m.start():]
1110        stripped_message = message[:m.start()].rstrip()
1111        return (stripped_message, jobtext)
1112
1113    def prepareLogMessage(self, template, message, jobs):
1114        """Edits the template returned from "p4 change -o" to insert
1115           the message in the Description field, and the jobs text in
1116           the Jobs field."""
1117        result = ""
1118
1119        inDescriptionSection = False
1120
1121        for line in template.split("\n"):
1122            if line.startswith("#"):
1123                result += line + "\n"
1124                continue
1125
1126            if inDescriptionSection:
1127                if line.startswith("Files:") or line.startswith("Jobs:"):
1128                    inDescriptionSection = False
1129                    # insert Jobs section
1130                    if jobs:
1131                        result += jobs + "\n"
1132                else:
1133                    continue
1134            else:
1135                if line.startswith("Description:"):
1136                    inDescriptionSection = True
1137                    line += "\n"
1138                    for messageLine in message.split("\n"):
1139                        line += "\t" + messageLine + "\n"
1140
1141            result += line + "\n"
1142
1143        return result
1144
1145    def patchRCSKeywords(self, file, pattern):
1146        # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
1147        (handle, outFileName) = tempfile.mkstemp(dir='.')
1148        try:
1149            outFile = os.fdopen(handle, "w+")
1150            inFile = open(file, "r")
1151            regexp = re.compile(pattern, re.VERBOSE)
1152            for line in inFile.readlines():
1153                line = regexp.sub(r'$\1$', line)
1154                outFile.write(line)
1155            inFile.close()
1156            outFile.close()
1157            # Forcibly overwrite the original file
1158            os.unlink(file)
1159            shutil.move(outFileName, file)
1160        except:
1161            # cleanup our temporary file
1162            os.unlink(outFileName)
1163            print "Failed to strip RCS keywords in %s" % file
1164            raise
1165
1166        print "Patched up RCS keywords in %s" % file
1167
1168    def p4UserForCommit(self,id):
1169        # Return the tuple (perforce user,git email) for a given git commit id
1170        self.getUserMapFromPerforceServer()
1171        gitEmail = read_pipe(["git", "log", "--max-count=1",
1172                              "--format=%ae", id])
1173        gitEmail = gitEmail.strip()
1174        if not self.emails.has_key(gitEmail):
1175            return (None,gitEmail)
1176        else:
1177            return (self.emails[gitEmail],gitEmail)
1178
1179    def checkValidP4Users(self,commits):
1180        # check if any git authors cannot be mapped to p4 users
1181        for id in commits:
1182            (user,email) = self.p4UserForCommit(id)
1183            if not user:
1184                msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
1185                if gitConfigBool("git-p4.allowMissingP4Users"):
1186                    print "%s" % msg
1187                else:
1188                    die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
1189
1190    def lastP4Changelist(self):
1191        # Get back the last changelist number submitted in this client spec. This
1192        # then gets used to patch up the username in the change. If the same
1193        # client spec is being used by multiple processes then this might go
1194        # wrong.
1195        results = p4CmdList("client -o")        # find the current client
1196        client = None
1197        for r in results:
1198            if r.has_key('Client'):
1199                client = r['Client']
1200                break
1201        if not client:
1202            die("could not get client spec")
1203        results = p4CmdList(["changes", "-c", client, "-m", "1"])
1204        for r in results:
1205            if r.has_key('change'):
1206                return r['change']
1207        die("Could not get changelist number for last submit - cannot patch up user details")
1208
1209    def modifyChangelistUser(self, changelist, newUser):
1210        # fixup the user field of a changelist after it has been submitted.
1211        changes = p4CmdList("change -o %s" % changelist)
1212        if len(changes) != 1:
1213            die("Bad output from p4 change modifying %s to user %s" %
1214                (changelist, newUser))
1215
1216        c = changes[0]
1217        if c['User'] == newUser: return   # nothing to do
1218        c['User'] = newUser
1219        input = marshal.dumps(c)
1220
1221        result = p4CmdList("change -f -i", stdin=input)
1222        for r in result:
1223            if r.has_key('code'):
1224                if r['code'] == 'error':
1225                    die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
1226            if r.has_key('data'):
1227                print("Updated user field for changelist %s to %s" % (changelist, newUser))
1228                return
1229        die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
1230
1231    def canChangeChangelists(self):
1232        # check to see if we have p4 admin or super-user permissions, either of
1233        # which are required to modify changelists.
1234        results = p4CmdList(["protects", self.depotPath])
1235        for r in results:
1236            if r.has_key('perm'):
1237                if r['perm'] == 'admin':
1238                    return 1
1239                if r['perm'] == 'super':
1240                    return 1
1241        return 0
1242
1243    def prepareSubmitTemplate(self):
1244        """Run "p4 change -o" to grab a change specification template.
1245           This does not use "p4 -G", as it is nice to keep the submission
1246           template in original order, since a human might edit it.
1247
1248           Remove lines in the Files section that show changes to files
1249           outside the depot path we're committing into."""
1250
1251        template = ""
1252        inFilesSection = False
1253        for line in p4_read_pipe_lines(['change', '-o']):
1254            if line.endswith("\r\n"):
1255                line = line[:-2] + "\n"
1256            if inFilesSection:
1257                if line.startswith("\t"):
1258                    # path starts and ends with a tab
1259                    path = line[1:]
1260                    lastTab = path.rfind("\t")
1261                    if lastTab != -1:
1262                        path = path[:lastTab]
1263                        if not p4PathStartsWith(path, self.depotPath):
1264                            continue
1265                else:
1266                    inFilesSection = False
1267            else:
1268                if line.startswith("Files:"):
1269                    inFilesSection = True
1270
1271            template += line
1272
1273        return template
1274
1275    def edit_template(self, template_file):
1276        """Invoke the editor to let the user change the submission
1277           message.  Return true if okay to continue with the submit."""
1278
1279        # if configured to skip the editing part, just submit
1280        if gitConfigBool("git-p4.skipSubmitEdit"):
1281            return True
1282
1283        # look at the modification time, to check later if the user saved
1284        # the file
1285        mtime = os.stat(template_file).st_mtime
1286
1287        # invoke the editor
1288        if os.environ.has_key("P4EDITOR") and (os.environ.get("P4EDITOR") != ""):
1289            editor = os.environ.get("P4EDITOR")
1290        else:
1291            editor = read_pipe("git var GIT_EDITOR").strip()
1292        system(["sh", "-c", ('%s "$@"' % editor), editor, template_file])
1293
1294        # If the file was not saved, prompt to see if this patch should
1295        # be skipped.  But skip this verification step if configured so.
1296        if gitConfigBool("git-p4.skipSubmitEditCheck"):
1297            return True
1298
1299        # modification time updated means user saved the file
1300        if os.stat(template_file).st_mtime > mtime:
1301            return True
1302
1303        while True:
1304            response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1305            if response == 'y':
1306                return True
1307            if response == 'n':
1308                return False
1309
1310    def get_diff_description(self, editedFiles, filesToAdd):
1311        # diff
1312        if os.environ.has_key("P4DIFF"):
1313            del(os.environ["P4DIFF"])
1314        diff = ""
1315        for editedFile in editedFiles:
1316            diff += p4_read_pipe(['diff', '-du',
1317                                  wildcard_encode(editedFile)])
1318
1319        # new file diff
1320        newdiff = ""
1321        for newFile in filesToAdd:
1322            newdiff += "==== new file ====\n"
1323            newdiff += "--- /dev/null\n"
1324            newdiff += "+++ %s\n" % newFile
1325            f = open(newFile, "r")
1326            for line in f.readlines():
1327                newdiff += "+" + line
1328            f.close()
1329
1330        return (diff + newdiff).replace('\r\n', '\n')
1331
1332    def applyCommit(self, id):
1333        """Apply one commit, return True if it succeeded."""
1334
1335        print "Applying", read_pipe(["git", "show", "-s",
1336                                     "--format=format:%h %s", id])
1337
1338        (p4User, gitEmail) = self.p4UserForCommit(id)
1339
1340        diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self.diffOpts, id, id))
1341        filesToAdd = set()
1342        filesToDelete = set()
1343        editedFiles = set()
1344        pureRenameCopy = set()
1345        filesToChangeExecBit = {}
1346
1347        for line in diff:
1348            diff = parseDiffTreeEntry(line)
1349            modifier = diff['status']
1350            path = diff['src']
1351            if modifier == "M":
1352                p4_edit(path)
1353                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1354                    filesToChangeExecBit[path] = diff['dst_mode']
1355                editedFiles.add(path)
1356            elif modifier == "A":
1357                filesToAdd.add(path)
1358                filesToChangeExecBit[path] = diff['dst_mode']
1359                if path in filesToDelete:
1360                    filesToDelete.remove(path)
1361            elif modifier == "D":
1362                filesToDelete.add(path)
1363                if path in filesToAdd:
1364                    filesToAdd.remove(path)
1365            elif modifier == "C":
1366                src, dest = diff['src'], diff['dst']
1367                p4_integrate(src, dest)
1368                pureRenameCopy.add(dest)
1369                if diff['src_sha1'] != diff['dst_sha1']:
1370                    p4_edit(dest)
1371                    pureRenameCopy.discard(dest)
1372                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1373                    p4_edit(dest)
1374                    pureRenameCopy.discard(dest)
1375                    filesToChangeExecBit[dest] = diff['dst_mode']
1376                if self.isWindows:
1377                    # turn off read-only attribute
1378                    os.chmod(dest, stat.S_IWRITE)
1379                os.unlink(dest)
1380                editedFiles.add(dest)
1381            elif modifier == "R":
1382                src, dest = diff['src'], diff['dst']
1383                if self.p4HasMoveCommand:
1384                    p4_edit(src)        # src must be open before move
1385                    p4_move(src, dest)  # opens for (move/delete, move/add)
1386                else:
1387                    p4_integrate(src, dest)
1388                    if diff['src_sha1'] != diff['dst_sha1']:
1389                        p4_edit(dest)
1390                    else:
1391                        pureRenameCopy.add(dest)
1392                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1393                    if not self.p4HasMoveCommand:
1394                        p4_edit(dest)   # with move: already open, writable
1395                    filesToChangeExecBit[dest] = diff['dst_mode']
1396                if not self.p4HasMoveCommand:
1397                    if self.isWindows:
1398                        os.chmod(dest, stat.S_IWRITE)
1399                    os.unlink(dest)
1400                    filesToDelete.add(src)
1401                editedFiles.add(dest)
1402            else:
1403                die("unknown modifier %s for %s" % (modifier, path))
1404
1405        diffcmd = "git diff-tree --full-index -p \"%s\"" % (id)
1406        patchcmd = diffcmd + " | git apply "
1407        tryPatchCmd = patchcmd + "--check -"
1408        applyPatchCmd = patchcmd + "--check --apply -"
1409        patch_succeeded = True
1410
1411        if os.system(tryPatchCmd) != 0:
1412            fixed_rcs_keywords = False
1413            patch_succeeded = False
1414            print "Unfortunately applying the change failed!"
1415
1416            # Patch failed, maybe it's just RCS keyword woes. Look through
1417            # the patch to see if that's possible.
1418            if gitConfigBool("git-p4.attemptRCSCleanup"):
1419                file = None
1420                pattern = None
1421                kwfiles = {}
1422                for file in editedFiles | filesToDelete:
1423                    # did this file's delta contain RCS keywords?
1424                    pattern = p4_keywords_regexp_for_file(file)
1425
1426                    if pattern:
1427                        # this file is a possibility...look for RCS keywords.
1428                        regexp = re.compile(pattern, re.VERBOSE)
1429                        for line in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1430                            if regexp.search(line):
1431                                if verbose:
1432                                    print "got keyword match on %s in %s in %s" % (pattern, line, file)
1433                                kwfiles[file] = pattern
1434                                break
1435
1436                for file in kwfiles:
1437                    if verbose:
1438                        print "zapping %s with %s" % (line,pattern)
1439                    # File is being deleted, so not open in p4.  Must
1440                    # disable the read-only bit on windows.
1441                    if self.isWindows and file not in editedFiles:
1442                        os.chmod(file, stat.S_IWRITE)
1443                    self.patchRCSKeywords(file, kwfiles[file])
1444                    fixed_rcs_keywords = True
1445
1446            if fixed_rcs_keywords:
1447                print "Retrying the patch with RCS keywords cleaned up"
1448                if os.system(tryPatchCmd) == 0:
1449                    patch_succeeded = True
1450
1451        if not patch_succeeded:
1452            for f in editedFiles:
1453                p4_revert(f)
1454            return False
1455
1456        #
1457        # Apply the patch for real, and do add/delete/+x handling.
1458        #
1459        system(applyPatchCmd)
1460
1461        for f in filesToAdd:
1462            p4_add(f)
1463        for f in filesToDelete:
1464            p4_revert(f)
1465            p4_delete(f)
1466
1467        # Set/clear executable bits
1468        for f in filesToChangeExecBit.keys():
1469            mode = filesToChangeExecBit[f]
1470            setP4ExecBit(f, mode)
1471
1472        #
1473        # Build p4 change description, starting with the contents
1474        # of the git commit message.
1475        #
1476        logMessage = extractLogMessageFromGitCommit(id)
1477        logMessage = logMessage.strip()
1478        (logMessage, jobs) = self.separate_jobs_from_description(logMessage)
1479
1480        template = self.prepareSubmitTemplate()
1481        submitTemplate = self.prepareLogMessage(template, logMessage, jobs)
1482
1483        if self.preserveUser:
1484           submitTemplate += "\n######## Actual user %s, modified after commit\n" % p4User
1485
1486        if self.checkAuthorship and not self.p4UserIsMe(p4User):
1487            submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
1488            submitTemplate += "######## Use option --preserve-user to modify authorship.\n"
1489            submitTemplate += "######## Variable git-p4.skipUserNameCheck hides this message.\n"
1490
1491        separatorLine = "######## everything below this line is just the diff #######\n"
1492        if not self.prepare_p4_only:
1493            submitTemplate += separatorLine
1494            submitTemplate += self.get_diff_description(editedFiles, filesToAdd)
1495
1496        (handle, fileName) = tempfile.mkstemp()
1497        tmpFile = os.fdopen(handle, "w+b")
1498        if self.isWindows:
1499            submitTemplate = submitTemplate.replace("\n", "\r\n")
1500        tmpFile.write(submitTemplate)
1501        tmpFile.close()
1502
1503        if self.prepare_p4_only:
1504            #
1505            # Leave the p4 tree prepared, and the submit template around
1506            # and let the user decide what to do next
1507            #
1508            print
1509            print "P4 workspace prepared for submission."
1510            print "To submit or revert, go to client workspace"
1511            print "  " + self.clientPath
1512            print
1513            print "To submit, use \"p4 submit\" to write a new description,"
1514            print "or \"p4 submit -i <%s\" to use the one prepared by" \
1515                  " \"git p4\"." % fileName
1516            print "You can delete the file \"%s\" when finished." % fileName
1517
1518            if self.preserveUser and p4User and not self.p4UserIsMe(p4User):
1519                print "To preserve change ownership by user %s, you must\n" \
1520                      "do \"p4 change -f <change>\" after submitting and\n" \
1521                      "edit the User field."
1522            if pureRenameCopy:
1523                print "After submitting, renamed files must be re-synced."
1524                print "Invoke \"p4 sync -f\" on each of these files:"
1525                for f in pureRenameCopy:
1526                    print "  " + f
1527
1528            print
1529            print "To revert the changes, use \"p4 revert ...\", and delete"
1530            print "the submit template file \"%s\"" % fileName
1531            if filesToAdd:
1532                print "Since the commit adds new files, they must be deleted:"
1533                for f in filesToAdd:
1534                    print "  " + f
1535            print
1536            return True
1537
1538        #
1539        # Let the user edit the change description, then submit it.
1540        #
1541        if self.edit_template(fileName):
1542            # read the edited message and submit
1543            ret = True
1544            tmpFile = open(fileName, "rb")
1545            message = tmpFile.read()
1546            tmpFile.close()
1547            if self.isWindows:
1548                message = message.replace("\r\n", "\n")
1549            submitTemplate = message[:message.index(separatorLine)]
1550            p4_write_pipe(['submit', '-i'], submitTemplate)
1551
1552            if self.preserveUser:
1553                if p4User:
1554                    # Get last changelist number. Cannot easily get it from
1555                    # the submit command output as the output is
1556                    # unmarshalled.
1557                    changelist = self.lastP4Changelist()
1558                    self.modifyChangelistUser(changelist, p4User)
1559
1560            # The rename/copy happened by applying a patch that created a
1561            # new file.  This leaves it writable, which confuses p4.
1562            for f in pureRenameCopy:
1563                p4_sync(f, "-f")
1564
1565        else:
1566            # skip this patch
1567            ret = False
1568            print "Submission cancelled, undoing p4 changes."
1569            for f in editedFiles:
1570                p4_revert(f)
1571            for f in filesToAdd:
1572                p4_revert(f)
1573                os.remove(f)
1574            for f in filesToDelete:
1575                p4_revert(f)
1576
1577        os.remove(fileName)
1578        return ret
1579
1580    # Export git tags as p4 labels. Create a p4 label and then tag
1581    # with that.
1582    def exportGitTags(self, gitTags):
1583        validLabelRegexp = gitConfig("git-p4.labelExportRegexp")
1584        if len(validLabelRegexp) == 0:
1585            validLabelRegexp = defaultLabelRegexp
1586        m = re.compile(validLabelRegexp)
1587
1588        for name in gitTags:
1589
1590            if not m.match(name):
1591                if verbose:
1592                    print "tag %s does not match regexp %s" % (name, validLabelRegexp)
1593                continue
1594
1595            # Get the p4 commit this corresponds to
1596            logMessage = extractLogMessageFromGitCommit(name)
1597            values = extractSettingsGitLog(logMessage)
1598
1599            if not values.has_key('change'):
1600                # a tag pointing to something not sent to p4; ignore
1601                if verbose:
1602                    print "git tag %s does not give a p4 commit" % name
1603                continue
1604            else:
1605                changelist = values['change']
1606
1607            # Get the tag details.
1608            inHeader = True
1609            isAnnotated = False
1610            body = []
1611            for l in read_pipe_lines(["git", "cat-file", "-p", name]):
1612                l = l.strip()
1613                if inHeader:
1614                    if re.match(r'tag\s+', l):
1615                        isAnnotated = True
1616                    elif re.match(r'\s*$', l):
1617                        inHeader = False
1618                        continue
1619                else:
1620                    body.append(l)
1621
1622            if not isAnnotated:
1623                body = ["lightweight tag imported by git p4\n"]
1624
1625            # Create the label - use the same view as the client spec we are using
1626            clientSpec = getClientSpec()
1627
1628            labelTemplate  = "Label: %s\n" % name
1629            labelTemplate += "Description:\n"
1630            for b in body:
1631                labelTemplate += "\t" + b + "\n"
1632            labelTemplate += "View:\n"
1633            for depot_side in clientSpec.mappings:
1634                labelTemplate += "\t%s\n" % depot_side
1635
1636            if self.dry_run:
1637                print "Would create p4 label %s for tag" % name
1638            elif self.prepare_p4_only:
1639                print "Not creating p4 label %s for tag due to option" \
1640                      " --prepare-p4-only" % name
1641            else:
1642                p4_write_pipe(["label", "-i"], labelTemplate)
1643
1644                # Use the label
1645                p4_system(["tag", "-l", name] +
1646                          ["%s@%s" % (depot_side, changelist) for depot_side in clientSpec.mappings])
1647
1648                if verbose:
1649                    print "created p4 label for tag %s" % name
1650
1651    def run(self, args):
1652        if len(args) == 0:
1653            self.master = currentGitBranch()
1654            if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
1655                die("Detecting current git branch failed!")
1656        elif len(args) == 1:
1657            self.master = args[0]
1658            if not branchExists(self.master):
1659                die("Branch %s does not exist" % self.master)
1660        else:
1661            return False
1662
1663        allowSubmit = gitConfig("git-p4.allowSubmit")
1664        if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
1665            die("%s is not in git-p4.allowSubmit" % self.master)
1666
1667        [upstream, settings] = findUpstreamBranchPoint()
1668        self.depotPath = settings['depot-paths'][0]
1669        if len(self.origin) == 0:
1670            self.origin = upstream
1671
1672        if self.preserveUser:
1673            if not self.canChangeChangelists():
1674                die("Cannot preserve user names without p4 super-user or admin permissions")
1675
1676        # if not set from the command line, try the config file
1677        if self.conflict_behavior is None:
1678            val = gitConfig("git-p4.conflict")
1679            if val:
1680                if val not in self.conflict_behavior_choices:
1681                    die("Invalid value '%s' for config git-p4.conflict" % val)
1682            else:
1683                val = "ask"
1684            self.conflict_behavior = val
1685
1686        if self.verbose:
1687            print "Origin branch is " + self.origin
1688
1689        if len(self.depotPath) == 0:
1690            print "Internal error: cannot locate perforce depot path from existing branches"
1691            sys.exit(128)
1692
1693        self.useClientSpec = False
1694        if gitConfigBool("git-p4.useclientspec"):
1695            self.useClientSpec = True
1696        if self.useClientSpec:
1697            self.clientSpecDirs = getClientSpec()
1698
1699        # Check for the existance of P4 branches
1700        branchesDetected = (len(p4BranchesInGit().keys()) > 1)
1701
1702        if self.useClientSpec and not branchesDetected:
1703            # all files are relative to the client spec
1704            self.clientPath = getClientRoot()
1705        else:
1706            self.clientPath = p4Where(self.depotPath)
1707
1708        if self.clientPath == "":
1709            die("Error: Cannot locate perforce checkout of %s in client view" % self.depotPath)
1710
1711        print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
1712        self.oldWorkingDirectory = os.getcwd()
1713
1714        # ensure the clientPath exists
1715        new_client_dir = False
1716        if not os.path.exists(self.clientPath):
1717            new_client_dir = True
1718            os.makedirs(self.clientPath)
1719
1720        chdir(self.clientPath, is_client_path=True)
1721        if self.dry_run:
1722            print "Would synchronize p4 checkout in %s" % self.clientPath
1723        else:
1724            print "Synchronizing p4 checkout..."
1725            if new_client_dir:
1726                # old one was destroyed, and maybe nobody told p4
1727                p4_sync("...", "-f")
1728            else:
1729                p4_sync("...")
1730        self.check()
1731
1732        commits = []
1733        for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self.origin, self.master)]):
1734            commits.append(line.strip())
1735        commits.reverse()
1736
1737        if self.preserveUser or gitConfigBool("git-p4.skipUserNameCheck"):
1738            self.checkAuthorship = False
1739        else:
1740            self.checkAuthorship = True
1741
1742        if self.preserveUser:
1743            self.checkValidP4Users(commits)
1744
1745        #
1746        # Build up a set of options to be passed to diff when
1747        # submitting each commit to p4.
1748        #
1749        if self.detectRenames:
1750            # command-line -M arg
1751            self.diffOpts = "-M"
1752        else:
1753            # If not explicitly set check the config variable
1754            detectRenames = gitConfig("git-p4.detectRenames")
1755
1756            if detectRenames.lower() == "false" or detectRenames == "":
1757                self.diffOpts = ""
1758            elif detectRenames.lower() == "true":
1759                self.diffOpts = "-M"
1760            else:
1761                self.diffOpts = "-M%s" % detectRenames
1762
1763        # no command-line arg for -C or --find-copies-harder, just
1764        # config variables
1765        detectCopies = gitConfig("git-p4.detectCopies")
1766        if detectCopies.lower() == "false" or detectCopies == "":
1767            pass
1768        elif detectCopies.lower() == "true":
1769            self.diffOpts += " -C"
1770        else:
1771            self.diffOpts += " -C%s" % detectCopies
1772
1773        if gitConfigBool("git-p4.detectCopiesHarder"):
1774            self.diffOpts += " --find-copies-harder"
1775
1776        #
1777        # Apply the commits, one at a time.  On failure, ask if should
1778        # continue to try the rest of the patches, or quit.
1779        #
1780        if self.dry_run:
1781            print "Would apply"
1782        applied = []
1783        last = len(commits) - 1
1784        for i, commit in enumerate(commits):
1785            if self.dry_run:
1786                print " ", read_pipe(["git", "show", "-s",
1787                                      "--format=format:%h %s", commit])
1788                ok = True
1789            else:
1790                ok = self.applyCommit(commit)
1791            if ok:
1792                applied.append(commit)
1793            else:
1794                if self.prepare_p4_only and i < last:
1795                    print "Processing only the first commit due to option" \
1796                          " --prepare-p4-only"
1797                    break
1798                if i < last:
1799                    quit = False
1800                    while True:
1801                        # prompt for what to do, or use the option/variable
1802                        if self.conflict_behavior == "ask":
1803                            print "What do you want to do?"
1804                            response = raw_input("[s]kip this commit but apply"
1805                                                 " the rest, or [q]uit? ")
1806                            if not response:
1807                                continue
1808                        elif self.conflict_behavior == "skip":
1809                            response = "s"
1810                        elif self.conflict_behavior == "quit":
1811                            response = "q"
1812                        else:
1813                            die("Unknown conflict_behavior '%s'" %
1814                                self.conflict_behavior)
1815
1816                        if response[0] == "s":
1817                            print "Skipping this commit, but applying the rest"
1818                            break
1819                        if response[0] == "q":
1820                            print "Quitting"
1821                            quit = True
1822                            break
1823                    if quit:
1824                        break
1825
1826        chdir(self.oldWorkingDirectory)
1827
1828        if self.dry_run:
1829            pass
1830        elif self.prepare_p4_only:
1831            pass
1832        elif len(commits) == len(applied):
1833            print "All commits applied!"
1834
1835            sync = P4Sync()
1836            if self.branch:
1837                sync.branch = self.branch
1838            sync.run([])
1839
1840            rebase = P4Rebase()
1841            rebase.rebase()
1842
1843        else:
1844            if len(applied) == 0:
1845                print "No commits applied."
1846            else:
1847                print "Applied only the commits marked with '*':"
1848                for c in commits:
1849                    if c in applied:
1850                        star = "*"
1851                    else:
1852                        star = " "
1853                    print star, read_pipe(["git", "show", "-s",
1854                                           "--format=format:%h %s",  c])
1855                print "You will have to do 'git p4 sync' and rebase."
1856
1857        if gitConfigBool("git-p4.exportLabels"):
1858            self.exportLabels = True
1859
1860        if self.exportLabels:
1861            p4Labels = getP4Labels(self.depotPath)
1862            gitTags = getGitTags()
1863
1864            missingGitTags = gitTags - p4Labels
1865            self.exportGitTags(missingGitTags)
1866
1867        # exit with error unless everything applied perfectly
1868        if len(commits) != len(applied):
1869                sys.exit(1)
1870
1871        return True
1872
1873class View(object):
1874    """Represent a p4 view ("p4 help views"), and map files in a
1875       repo according to the view."""
1876
1877    def __init__(self, client_name):
1878        self.mappings = []
1879        self.client_prefix = "//%s/" % client_name
1880        # cache results of "p4 where" to lookup client file locations
1881        self.client_spec_path_cache = {}
1882
1883    def append(self, view_line):
1884        """Parse a view line, splitting it into depot and client
1885           sides.  Append to self.mappings, preserving order.  This
1886           is only needed for tag creation."""
1887
1888        # Split the view line into exactly two words.  P4 enforces
1889        # structure on these lines that simplifies this quite a bit.
1890        #
1891        # Either or both words may be double-quoted.
1892        # Single quotes do not matter.
1893        # Double-quote marks cannot occur inside the words.
1894        # A + or - prefix is also inside the quotes.
1895        # There are no quotes unless they contain a space.
1896        # The line is already white-space stripped.
1897        # The two words are separated by a single space.
1898        #
1899        if view_line[0] == '"':
1900            # First word is double quoted.  Find its end.
1901            close_quote_index = view_line.find('"', 1)
1902            if close_quote_index <= 0:
1903                die("No first-word closing quote found: %s" % view_line)
1904            depot_side = view_line[1:close_quote_index]
1905            # skip closing quote and space
1906            rhs_index = close_quote_index + 1 + 1
1907        else:
1908            space_index = view_line.find(" ")
1909            if space_index <= 0:
1910                die("No word-splitting space found: %s" % view_line)
1911            depot_side = view_line[0:space_index]
1912            rhs_index = space_index + 1
1913
1914        # prefix + means overlay on previous mapping
1915        if depot_side.startswith("+"):
1916            depot_side = depot_side[1:]
1917
1918        # prefix - means exclude this path, leave out of mappings
1919        exclude = False
1920        if depot_side.startswith("-"):
1921            exclude = True
1922            depot_side = depot_side[1:]
1923
1924        if not exclude:
1925            self.mappings.append(depot_side)
1926
1927    def convert_client_path(self, clientFile):
1928        # chop off //client/ part to make it relative
1929        if not clientFile.startswith(self.client_prefix):
1930            die("No prefix '%s' on clientFile '%s'" %
1931                (self.client_prefix, clientFile))
1932        return clientFile[len(self.client_prefix):]
1933
1934    def update_client_spec_path_cache(self, files):
1935        """ Caching file paths by "p4 where" batch query """
1936
1937        # List depot file paths exclude that already cached
1938        fileArgs = [f['path'] for f in files if f['path'] not in self.client_spec_path_cache]
1939
1940        if len(fileArgs) == 0:
1941            return  # All files in cache
1942
1943        where_result = p4CmdList(["-x", "-", "where"], stdin=fileArgs)
1944        for res in where_result:
1945            if "code" in res and res["code"] == "error":
1946                # assume error is "... file(s) not in client view"
1947                continue
1948            if "clientFile" not in res:
1949                die("No clientFile in 'p4 where' output")
1950            if "unmap" in res:
1951                # it will list all of them, but only one not unmap-ped
1952                continue
1953            self.client_spec_path_cache[res['depotFile']] = self.convert_client_path(res["clientFile"])
1954
1955        # not found files or unmap files set to ""
1956        for depotFile in fileArgs:
1957            if depotFile not in self.client_spec_path_cache:
1958                self.client_spec_path_cache[depotFile] = ""
1959
1960    def map_in_client(self, depot_path):
1961        """Return the relative location in the client where this
1962           depot file should live.  Returns "" if the file should
1963           not be mapped in the client."""
1964
1965        if depot_path in self.client_spec_path_cache:
1966            return self.client_spec_path_cache[depot_path]
1967
1968        die( "Error: %s is not found in client spec path" % depot_path )
1969        return ""
1970
1971class P4Sync(Command, P4UserMap):
1972    delete_actions = ( "delete", "move/delete", "purge" )
1973
1974    def __init__(self):
1975        Command.__init__(self)
1976        P4UserMap.__init__(self)
1977        self.options = [
1978                optparse.make_option("--branch", dest="branch"),
1979                optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
1980                optparse.make_option("--changesfile", dest="changesFile"),
1981                optparse.make_option("--silent", dest="silent", action="store_true"),
1982                optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
1983                optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
1984                optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
1985                                     help="Import into refs/heads/ , not refs/remotes"),
1986                optparse.make_option("--max-changes", dest="maxChanges",
1987                                     help="Maximum number of changes to import"),
1988                optparse.make_option("--changes-block-size", dest="changes_block_size", type="int",
1989                                     help="Internal block size to use when iteratively calling p4 changes"),
1990                optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
1991                                     help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
1992                optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
1993                                     help="Only sync files that are included in the Perforce Client Spec"),
1994                optparse.make_option("-/", dest="cloneExclude",
1995                                     action="append", type="string",
1996                                     help="exclude depot path"),
1997        ]
1998        self.description = """Imports from Perforce into a git repository.\n
1999    example:
2000    //depot/my/project/ -- to import the current head
2001    //depot/my/project/@all -- to import everything
2002    //depot/my/project/@1,6 -- to import only from revision 1 to 6
2003
2004    (a ... is not needed in the path p4 specification, it's added implicitly)"""
2005
2006        self.usage += " //depot/path[@revRange]"
2007        self.silent = False
2008        self.createdBranches = set()
2009        self.committedChanges = set()
2010        self.branch = ""
2011        self.detectBranches = False
2012        self.detectLabels = False
2013        self.importLabels = False
2014        self.changesFile = ""
2015        self.syncWithOrigin = True
2016        self.importIntoRemotes = True
2017        self.maxChanges = ""
2018        self.changes_block_size = None
2019        self.keepRepoPath = False
2020        self.depotPaths = None
2021        self.p4BranchesInGit = []
2022        self.cloneExclude = []
2023        self.useClientSpec = False
2024        self.useClientSpec_from_options = False
2025        self.clientSpecDirs = None
2026        self.tempBranches = []
2027        self.tempBranchLocation = "git-p4-tmp"
2028
2029        if gitConfig("git-p4.syncFromOrigin") == "false":
2030            self.syncWithOrigin = False
2031
2032    # This is required for the "append" cloneExclude action
2033    def ensure_value(self, attr, value):
2034        if not hasattr(self, attr) or getattr(self, attr) is None:
2035            setattr(self, attr, value)
2036        return getattr(self, attr)
2037
2038    # Force a checkpoint in fast-import and wait for it to finish
2039    def checkpoint(self):
2040        self.gitStream.write("checkpoint\n\n")
2041        self.gitStream.write("progress checkpoint\n\n")
2042        out = self.gitOutput.readline()
2043        if self.verbose:
2044            print "checkpoint finished: " + out
2045
2046    def extractFilesFromCommit(self, commit):
2047        self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
2048                             for path in self.cloneExclude]
2049        files = []
2050        fnum = 0
2051        while commit.has_key("depotFile%s" % fnum):
2052            path =  commit["depotFile%s" % fnum]
2053
2054            if [p for p in self.cloneExclude
2055                if p4PathStartsWith(path, p)]:
2056                found = False
2057            else:
2058                found = [p for p in self.depotPaths
2059                         if p4PathStartsWith(path, p)]
2060            if not found:
2061                fnum = fnum + 1
2062                continue
2063
2064            file = {}
2065            file["path"] = path
2066            file["rev"] = commit["rev%s" % fnum]
2067            file["action"] = commit["action%s" % fnum]
2068            file["type"] = commit["type%s" % fnum]
2069            files.append(file)
2070            fnum = fnum + 1
2071        return files
2072
2073    def stripRepoPath(self, path, prefixes):
2074        """When streaming files, this is called to map a p4 depot path
2075           to where it should go in git.  The prefixes are either
2076           self.depotPaths, or self.branchPrefixes in the case of
2077           branch detection."""
2078
2079        if self.useClientSpec:
2080            # branch detection moves files up a level (the branch name)
2081            # from what client spec interpretation gives
2082            path = self.clientSpecDirs.map_in_client(path)
2083            if self.detectBranches:
2084                for b in self.knownBranches:
2085                    if path.startswith(b + "/"):
2086                        path = path[len(b)+1:]
2087
2088        elif self.keepRepoPath:
2089            # Preserve everything in relative path name except leading
2090            # //depot/; just look at first prefix as they all should
2091            # be in the same depot.
2092            depot = re.sub("^(//[^/]+/).*", r'\1', prefixes[0])
2093            if p4PathStartsWith(path, depot):
2094                path = path[len(depot):]
2095
2096        else:
2097            for p in prefixes:
2098                if p4PathStartsWith(path, p):
2099                    path = path[len(p):]
2100                    break
2101
2102        path = wildcard_decode(path)
2103        return path
2104
2105    def splitFilesIntoBranches(self, commit):
2106        """Look at each depotFile in the commit to figure out to what
2107           branch it belongs."""
2108
2109        if self.clientSpecDirs:
2110            files = self.extractFilesFromCommit(commit)
2111            self.clientSpecDirs.update_client_spec_path_cache(files)
2112
2113        branches = {}
2114        fnum = 0
2115        while commit.has_key("depotFile%s" % fnum):
2116            path =  commit["depotFile%s" % fnum]
2117            found = [p for p in self.depotPaths
2118                     if p4PathStartsWith(path, p)]
2119            if not found:
2120                fnum = fnum + 1
2121                continue
2122
2123            file = {}
2124            file["path"] = path
2125            file["rev"] = commit["rev%s" % fnum]
2126            file["action"] = commit["action%s" % fnum]
2127            file["type"] = commit["type%s" % fnum]
2128            fnum = fnum + 1
2129
2130            # start with the full relative path where this file would
2131            # go in a p4 client
2132            if self.useClientSpec:
2133                relPath = self.clientSpecDirs.map_in_client(path)
2134            else:
2135                relPath = self.stripRepoPath(path, self.depotPaths)
2136
2137            for branch in self.knownBranches.keys():
2138                # add a trailing slash so that a commit into qt/4.2foo
2139                # doesn't end up in qt/4.2, e.g.
2140                if relPath.startswith(branch + "/"):
2141                    if branch not in branches:
2142                        branches[branch] = []
2143                    branches[branch].append(file)
2144                    break
2145
2146        return branches
2147
2148    # output one file from the P4 stream
2149    # - helper for streamP4Files
2150
2151    def streamOneP4File(self, file, contents):
2152        relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
2153        if verbose:
2154            sys.stderr.write("%s\n" % relPath)
2155
2156        (type_base, type_mods) = split_p4_type(file["type"])
2157
2158        git_mode = "100644"
2159        if "x" in type_mods:
2160            git_mode = "100755"
2161        if type_base == "symlink":
2162            git_mode = "120000"
2163            # p4 print on a symlink sometimes contains "target\n";
2164            # if it does, remove the newline
2165            data = ''.join(contents)
2166            if not data:
2167                # Some version of p4 allowed creating a symlink that pointed
2168                # to nothing.  This causes p4 errors when checking out such
2169                # a change, and errors here too.  Work around it by ignoring
2170                # the bad symlink; hopefully a future change fixes it.
2171                print "\nIgnoring empty symlink in %s" % file['depotFile']
2172                return
2173            elif data[-1] == '\n':
2174                contents = [data[:-1]]
2175            else:
2176                contents = [data]
2177
2178        if type_base == "utf16":
2179            # p4 delivers different text in the python output to -G
2180            # than it does when using "print -o", or normal p4 client
2181            # operations.  utf16 is converted to ascii or utf8, perhaps.
2182            # But ascii text saved as -t utf16 is completely mangled.
2183            # Invoke print -o to get the real contents.
2184            #
2185            # On windows, the newlines will always be mangled by print, so put
2186            # them back too.  This is not needed to the cygwin windows version,
2187            # just the native "NT" type.
2188            #
2189            text = p4_read_pipe(['print', '-q', '-o', '-', "%s@%s" % (file['depotFile'], file['change']) ])
2190            if p4_version_string().find("/NT") >= 0:
2191                text = text.replace("\r\n", "\n")
2192            contents = [ text ]
2193
2194        if type_base == "apple":
2195            # Apple filetype files will be streamed as a concatenation of
2196            # its appledouble header and the contents.  This is useless
2197            # on both macs and non-macs.  If using "print -q -o xx", it
2198            # will create "xx" with the data, and "%xx" with the header.
2199            # This is also not very useful.
2200            #
2201            # Ideally, someday, this script can learn how to generate
2202            # appledouble files directly and import those to git, but
2203            # non-mac machines can never find a use for apple filetype.
2204            print "\nIgnoring apple filetype file %s" % file['depotFile']
2205            return
2206
2207        # Note that we do not try to de-mangle keywords on utf16 files,
2208        # even though in theory somebody may want that.
2209        pattern = p4_keywords_regexp_for_type(type_base, type_mods)
2210        if pattern:
2211            regexp = re.compile(pattern, re.VERBOSE)
2212            text = ''.join(contents)
2213            text = regexp.sub(r'$\1$', text)
2214            contents = [ text ]
2215
2216        try:
2217            relPath.decode('ascii')
2218        except:
2219            encoding = 'utf8'
2220            if gitConfig('git-p4.pathEncoding'):
2221                encoding = gitConfig('git-p4.pathEncoding')
2222            relPath = relPath.decode(encoding).encode('utf8', 'replace')
2223            if self.verbose:
2224                print 'Path with non-ASCII characters detected. Used %s to encode: %s ' % (encoding, relPath)
2225
2226        self.gitStream.write("M %s inline %s\n" % (git_mode, relPath))
2227
2228        # total length...
2229        length = 0
2230        for d in contents:
2231            length = length + len(d)
2232
2233        self.gitStream.write("data %d\n" % length)
2234        for d in contents:
2235            self.gitStream.write(d)
2236        self.gitStream.write("\n")
2237
2238    def streamOneP4Deletion(self, file):
2239        relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
2240        if verbose:
2241            sys.stderr.write("delete %s\n" % relPath)
2242        self.gitStream.write("D %s\n" % relPath)
2243
2244    # handle another chunk of streaming data
2245    def streamP4FilesCb(self, marshalled):
2246
2247        # catch p4 errors and complain
2248        err = None
2249        if "code" in marshalled:
2250            if marshalled["code"] == "error":
2251                if "data" in marshalled:
2252                    err = marshalled["data"].rstrip()
2253        if err:
2254            f = None
2255            if self.stream_have_file_info:
2256                if "depotFile" in self.stream_file:
2257                    f = self.stream_file["depotFile"]
2258            # force a failure in fast-import, else an empty
2259            # commit will be made
2260            self.gitStream.write("\n")
2261            self.gitStream.write("die-now\n")
2262            self.gitStream.close()
2263            # ignore errors, but make sure it exits first
2264            self.importProcess.wait()
2265            if f:
2266                die("Error from p4 print for %s: %s" % (f, err))
2267            else:
2268                die("Error from p4 print: %s" % err)
2269
2270        if marshalled.has_key('depotFile') and self.stream_have_file_info:
2271            # start of a new file - output the old one first
2272            self.streamOneP4File(self.stream_file, self.stream_contents)
2273            self.stream_file = {}
2274            self.stream_contents = []
2275            self.stream_have_file_info = False
2276
2277        # pick up the new file information... for the
2278        # 'data' field we need to append to our array
2279        for k in marshalled.keys():
2280            if k == 'data':
2281                self.stream_contents.append(marshalled['data'])
2282            else:
2283                self.stream_file[k] = marshalled[k]
2284
2285        self.stream_have_file_info = True
2286
2287    # Stream directly from "p4 files" into "git fast-import"
2288    def streamP4Files(self, files):
2289        filesForCommit = []
2290        filesToRead = []
2291        filesToDelete = []
2292
2293        for f in files:
2294            # if using a client spec, only add the files that have
2295            # a path in the client
2296            if self.clientSpecDirs:
2297                if self.clientSpecDirs.map_in_client(f['path']) == "":
2298                    continue
2299
2300            filesForCommit.append(f)
2301            if f['action'] in self.delete_actions:
2302                filesToDelete.append(f)
2303            else:
2304                filesToRead.append(f)
2305
2306        # deleted files...
2307        for f in filesToDelete:
2308            self.streamOneP4Deletion(f)
2309
2310        if len(filesToRead) > 0:
2311            self.stream_file = {}
2312            self.stream_contents = []
2313            self.stream_have_file_info = False
2314
2315            # curry self argument
2316            def streamP4FilesCbSelf(entry):
2317                self.streamP4FilesCb(entry)
2318
2319            fileArgs = ['%s#%s' % (f['path'], f['rev']) for f in filesToRead]
2320
2321            p4CmdList(["-x", "-", "print"],
2322                      stdin=fileArgs,
2323                      cb=streamP4FilesCbSelf)
2324
2325            # do the last chunk
2326            if self.stream_file.has_key('depotFile'):
2327                self.streamOneP4File(self.stream_file, self.stream_contents)
2328
2329    def make_email(self, userid):
2330        if userid in self.users:
2331            return self.users[userid]
2332        else:
2333            return "%s <a@b>" % userid
2334
2335    # Stream a p4 tag
2336    def streamTag(self, gitStream, labelName, labelDetails, commit, epoch):
2337        if verbose:
2338            print "writing tag %s for commit %s" % (labelName, commit)
2339        gitStream.write("tag %s\n" % labelName)
2340        gitStream.write("from %s\n" % commit)
2341
2342        if labelDetails.has_key('Owner'):
2343            owner = labelDetails["Owner"]
2344        else:
2345            owner = None
2346
2347        # Try to use the owner of the p4 label, or failing that,
2348        # the current p4 user id.
2349        if owner:
2350            email = self.make_email(owner)
2351        else:
2352            email = self.make_email(self.p4UserId())
2353        tagger = "%s %s %s" % (email, epoch, self.tz)
2354
2355        gitStream.write("tagger %s\n" % tagger)
2356
2357        print "labelDetails=",labelDetails
2358        if labelDetails.has_key('Description'):
2359            description = labelDetails['Description']
2360        else:
2361            description = 'Label from git p4'
2362
2363        gitStream.write("data %d\n" % len(description))
2364        gitStream.write(description)
2365        gitStream.write("\n")
2366
2367    def commit(self, details, files, branch, parent = ""):
2368        epoch = details["time"]
2369        author = details["user"]
2370
2371        if self.verbose:
2372            print "commit into %s" % branch
2373
2374        # start with reading files; if that fails, we should not
2375        # create a commit.
2376        new_files = []
2377        for f in files:
2378            if [p for p in self.branchPrefixes if p4PathStartsWith(f['path'], p)]:
2379                new_files.append (f)
2380            else:
2381                sys.stderr.write("Ignoring file outside of prefix: %s\n" % f['path'])
2382
2383        if self.clientSpecDirs:
2384            self.clientSpecDirs.update_client_spec_path_cache(files)
2385
2386        self.gitStream.write("commit %s\n" % branch)
2387#        gitStream.write("mark :%s\n" % details["change"])
2388        self.committedChanges.add(int(details["change"]))
2389        committer = ""
2390        if author not in self.users:
2391            self.getUserMapFromPerforceServer()
2392        committer = "%s %s %s" % (self.make_email(author), epoch, self.tz)
2393
2394        self.gitStream.write("committer %s\n" % committer)
2395
2396        self.gitStream.write("data <<EOT\n")
2397        self.gitStream.write(details["desc"])
2398        self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
2399                             (','.join(self.branchPrefixes), details["change"]))
2400        if len(details['options']) > 0:
2401            self.gitStream.write(": options = %s" % details['options'])
2402        self.gitStream.write("]\nEOT\n\n")
2403
2404        if len(parent) > 0:
2405            if self.verbose:
2406                print "parent %s" % parent
2407            self.gitStream.write("from %s\n" % parent)
2408
2409        self.streamP4Files(new_files)
2410        self.gitStream.write("\n")
2411
2412        change = int(details["change"])
2413
2414        if self.labels.has_key(change):
2415            label = self.labels[change]
2416            labelDetails = label[0]
2417            labelRevisions = label[1]
2418            if self.verbose:
2419                print "Change %s is labelled %s" % (change, labelDetails)
2420
2421            files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
2422                                                for p in self.branchPrefixes])
2423
2424            if len(files) == len(labelRevisions):
2425
2426                cleanedFiles = {}
2427                for info in files:
2428                    if info["action"] in self.delete_actions:
2429                        continue
2430                    cleanedFiles[info["depotFile"]] = info["rev"]
2431
2432                if cleanedFiles == labelRevisions:
2433                    self.streamTag(self.gitStream, 'tag_%s' % labelDetails['label'], labelDetails, branch, epoch)
2434
2435                else:
2436                    if not self.silent:
2437                        print ("Tag %s does not match with change %s: files do not match."
2438                               % (labelDetails["label"], change))
2439
2440            else:
2441                if not self.silent:
2442                    print ("Tag %s does not match with change %s: file count is different."
2443                           % (labelDetails["label"], change))
2444
2445    # Build a dictionary of changelists and labels, for "detect-labels" option.
2446    def getLabels(self):
2447        self.labels = {}
2448
2449        l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths])
2450        if len(l) > 0 and not self.silent:
2451            print "Finding files belonging to labels in %s" % `self.depotPaths`
2452
2453        for output in l:
2454            label = output["label"]
2455            revisions = {}
2456            newestChange = 0
2457            if self.verbose:
2458                print "Querying files for label %s" % label
2459            for file in p4CmdList(["files"] +
2460                                      ["%s...@%s" % (p, label)
2461                                          for p in self.depotPaths]):
2462                revisions[file["depotFile"]] = file["rev"]
2463                change = int(file["change"])
2464                if change > newestChange:
2465                    newestChange = change
2466
2467            self.labels[newestChange] = [output, revisions]
2468
2469        if self.verbose:
2470            print "Label changes: %s" % self.labels.keys()
2471
2472    # Import p4 labels as git tags. A direct mapping does not
2473    # exist, so assume that if all the files are at the same revision
2474    # then we can use that, or it's something more complicated we should
2475    # just ignore.
2476    def importP4Labels(self, stream, p4Labels):
2477        if verbose:
2478            print "import p4 labels: " + ' '.join(p4Labels)
2479
2480        ignoredP4Labels = gitConfigList("git-p4.ignoredP4Labels")
2481        validLabelRegexp = gitConfig("git-p4.labelImportRegexp")
2482        if len(validLabelRegexp) == 0:
2483            validLabelRegexp = defaultLabelRegexp
2484        m = re.compile(validLabelRegexp)
2485
2486        for name in p4Labels:
2487            commitFound = False
2488
2489            if not m.match(name):
2490                if verbose:
2491                    print "label %s does not match regexp %s" % (name,validLabelRegexp)
2492                continue
2493
2494            if name in ignoredP4Labels:
2495                continue
2496
2497            labelDetails = p4CmdList(['label', "-o", name])[0]
2498
2499            # get the most recent changelist for each file in this label
2500            change = p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p, name)
2501                                for p in self.depotPaths])
2502
2503            if change.has_key('change'):
2504                # find the corresponding git commit; take the oldest commit
2505                changelist = int(change['change'])
2506                gitCommit = read_pipe(["git", "rev-list", "--max-count=1",
2507                     "--reverse", ":/\[git-p4:.*change = %d\]" % changelist])
2508                if len(gitCommit) == 0:
2509                    print "could not find git commit for changelist %d" % changelist
2510                else:
2511                    gitCommit = gitCommit.strip()
2512                    commitFound = True
2513                    # Convert from p4 time format
2514                    try:
2515                        tmwhen = time.strptime(labelDetails['Update'], "%Y/%m/%d %H:%M:%S")
2516                    except ValueError:
2517                        print "Could not convert label time %s" % labelDetails['Update']
2518                        tmwhen = 1
2519
2520                    when = int(time.mktime(tmwhen))
2521                    self.streamTag(stream, name, labelDetails, gitCommit, when)
2522                    if verbose:
2523                        print "p4 label %s mapped to git commit %s" % (name, gitCommit)
2524            else:
2525                if verbose:
2526                    print "Label %s has no changelists - possibly deleted?" % name
2527
2528            if not commitFound:
2529                # We can't import this label; don't try again as it will get very
2530                # expensive repeatedly fetching all the files for labels that will
2531                # never be imported. If the label is moved in the future, the
2532                # ignore will need to be removed manually.
2533                system(["git", "config", "--add", "git-p4.ignoredP4Labels", name])
2534
2535    def guessProjectName(self):
2536        for p in self.depotPaths:
2537            if p.endswith("/"):
2538                p = p[:-1]
2539            p = p[p.strip().rfind("/") + 1:]
2540            if not p.endswith("/"):
2541               p += "/"
2542            return p
2543
2544    def getBranchMapping(self):
2545        lostAndFoundBranches = set()
2546
2547        user = gitConfig("git-p4.branchUser")
2548        if len(user) > 0:
2549            command = "branches -u %s" % user
2550        else:
2551            command = "branches"
2552
2553        for info in p4CmdList(command):
2554            details = p4Cmd(["branch", "-o", info["branch"]])
2555            viewIdx = 0
2556            while details.has_key("View%s" % viewIdx):
2557                paths = details["View%s" % viewIdx].split(" ")
2558                viewIdx = viewIdx + 1
2559                # require standard //depot/foo/... //depot/bar/... mapping
2560                if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
2561                    continue
2562                source = paths[0]
2563                destination = paths[1]
2564                ## HACK
2565                if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
2566                    source = source[len(self.depotPaths[0]):-4]
2567                    destination = destination[len(self.depotPaths[0]):-4]
2568
2569                    if destination in self.knownBranches:
2570                        if not self.silent:
2571                            print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
2572                            print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
2573                        continue
2574
2575                    self.knownBranches[destination] = source
2576
2577                    lostAndFoundBranches.discard(destination)
2578
2579                    if source not in self.knownBranches:
2580                        lostAndFoundBranches.add(source)
2581
2582        # Perforce does not strictly require branches to be defined, so we also
2583        # check git config for a branch list.
2584        #
2585        # Example of branch definition in git config file:
2586        # [git-p4]
2587        #   branchList=main:branchA
2588        #   branchList=main:branchB
2589        #   branchList=branchA:branchC
2590        configBranches = gitConfigList("git-p4.branchList")
2591        for branch in configBranches:
2592            if branch:
2593                (source, destination) = branch.split(":")
2594                self.knownBranches[destination] = source
2595
2596                lostAndFoundBranches.discard(destination)
2597
2598                if source not in self.knownBranches:
2599                    lostAndFoundBranches.add(source)
2600
2601
2602        for branch in lostAndFoundBranches:
2603            self.knownBranches[branch] = branch
2604
2605    def getBranchMappingFromGitBranches(self):
2606        branches = p4BranchesInGit(self.importIntoRemotes)
2607        for branch in branches.keys():
2608            if branch == "master":
2609                branch = "main"
2610            else:
2611                branch = branch[len(self.projectName):]
2612            self.knownBranches[branch] = branch
2613
2614    def updateOptionDict(self, d):
2615        option_keys = {}
2616        if self.keepRepoPath:
2617            option_keys['keepRepoPath'] = 1
2618
2619        d["options"] = ' '.join(sorted(option_keys.keys()))
2620
2621    def readOptions(self, d):
2622        self.keepRepoPath = (d.has_key('options')
2623                             and ('keepRepoPath' in d['options']))
2624
2625    def gitRefForBranch(self, branch):
2626        if branch == "main":
2627            return self.refPrefix + "master"
2628
2629        if len(branch) <= 0:
2630            return branch
2631
2632        return self.refPrefix + self.projectName + branch
2633
2634    def gitCommitByP4Change(self, ref, change):
2635        if self.verbose:
2636            print "looking in ref " + ref + " for change %s using bisect..." % change
2637
2638        earliestCommit = ""
2639        latestCommit = parseRevision(ref)
2640
2641        while True:
2642            if self.verbose:
2643                print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
2644            next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
2645            if len(next) == 0:
2646                if self.verbose:
2647                    print "argh"
2648                return ""
2649            log = extractLogMessageFromGitCommit(next)
2650            settings = extractSettingsGitLog(log)
2651            currentChange = int(settings['change'])
2652            if self.verbose:
2653                print "current change %s" % currentChange
2654
2655            if currentChange == change:
2656                if self.verbose:
2657                    print "found %s" % next
2658                return next
2659
2660            if currentChange < change:
2661                earliestCommit = "^%s" % next
2662            else:
2663                latestCommit = "%s" % next
2664
2665        return ""
2666
2667    def importNewBranch(self, branch, maxChange):
2668        # make fast-import flush all changes to disk and update the refs using the checkpoint
2669        # command so that we can try to find the branch parent in the git history
2670        self.gitStream.write("checkpoint\n\n");
2671        self.gitStream.flush();
2672        branchPrefix = self.depotPaths[0] + branch + "/"
2673        range = "@1,%s" % maxChange
2674        #print "prefix" + branchPrefix
2675        changes = p4ChangesForPaths([branchPrefix], range, self.changes_block_size)
2676        if len(changes) <= 0:
2677            return False
2678        firstChange = changes[0]
2679        #print "first change in branch: %s" % firstChange
2680        sourceBranch = self.knownBranches[branch]
2681        sourceDepotPath = self.depotPaths[0] + sourceBranch
2682        sourceRef = self.gitRefForBranch(sourceBranch)
2683        #print "source " + sourceBranch
2684
2685        branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"])
2686        #print "branch parent: %s" % branchParentChange
2687        gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
2688        if len(gitParent) > 0:
2689            self.initialParents[self.gitRefForBranch(branch)] = gitParent
2690            #print "parent git commit: %s" % gitParent
2691
2692        self.importChanges(changes)
2693        return True
2694
2695    def searchParent(self, parent, branch, target):
2696        parentFound = False
2697        for blob in read_pipe_lines(["git", "rev-list", "--reverse",
2698                                     "--no-merges", parent]):
2699            blob = blob.strip()
2700            if len(read_pipe(["git", "diff-tree", blob, target])) == 0:
2701                parentFound = True
2702                if self.verbose:
2703                    print "Found parent of %s in commit %s" % (branch, blob)
2704                break
2705        if parentFound:
2706            return blob
2707        else:
2708            return None
2709
2710    def importChanges(self, changes):
2711        cnt = 1
2712        for change in changes:
2713            description = p4_describe(change)
2714            self.updateOptionDict(description)
2715
2716            if not self.silent:
2717                sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
2718                sys.stdout.flush()
2719            cnt = cnt + 1
2720
2721            try:
2722                if self.detectBranches:
2723                    branches = self.splitFilesIntoBranches(description)
2724                    for branch in branches.keys():
2725                        ## HACK  --hwn
2726                        branchPrefix = self.depotPaths[0] + branch + "/"
2727                        self.branchPrefixes = [ branchPrefix ]
2728
2729                        parent = ""
2730
2731                        filesForCommit = branches[branch]
2732
2733                        if self.verbose:
2734                            print "branch is %s" % branch
2735
2736                        self.updatedBranches.add(branch)
2737
2738                        if branch not in self.createdBranches:
2739                            self.createdBranches.add(branch)
2740                            parent = self.knownBranches[branch]
2741                            if parent == branch:
2742                                parent = ""
2743                            else:
2744                                fullBranch = self.projectName + branch
2745                                if fullBranch not in self.p4BranchesInGit:
2746                                    if not self.silent:
2747                                        print("\n    Importing new branch %s" % fullBranch);
2748                                    if self.importNewBranch(branch, change - 1):
2749                                        parent = ""
2750                                        self.p4BranchesInGit.append(fullBranch)
2751                                    if not self.silent:
2752                                        print("\n    Resuming with change %s" % change);
2753
2754                                if self.verbose:
2755                                    print "parent determined through known branches: %s" % parent
2756
2757                        branch = self.gitRefForBranch(branch)
2758                        parent = self.gitRefForBranch(parent)
2759
2760                        if self.verbose:
2761                            print "looking for initial parent for %s; current parent is %s" % (branch, parent)
2762
2763                        if len(parent) == 0 and branch in self.initialParents:
2764                            parent = self.initialParents[branch]
2765                            del self.initialParents[branch]
2766
2767                        blob = None
2768                        if len(parent) > 0:
2769                            tempBranch = "%s/%d" % (self.tempBranchLocation, change)
2770                            if self.verbose:
2771                                print "Creating temporary branch: " + tempBranch
2772                            self.commit(description, filesForCommit, tempBranch)
2773                            self.tempBranches.append(tempBranch)
2774                            self.checkpoint()
2775                            blob = self.searchParent(parent, branch, tempBranch)
2776                        if blob:
2777                            self.commit(description, filesForCommit, branch, blob)
2778                        else:
2779                            if self.verbose:
2780                                print "Parent of %s not found. Committing into head of %s" % (branch, parent)
2781                            self.commit(description, filesForCommit, branch, parent)
2782                else:
2783                    files = self.extractFilesFromCommit(description)
2784                    self.commit(description, files, self.branch,
2785                                self.initialParent)
2786                    # only needed once, to connect to the previous commit
2787                    self.initialParent = ""
2788            except IOError:
2789                print self.gitError.read()
2790                sys.exit(1)
2791
2792    def importHeadRevision(self, revision):
2793        print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
2794
2795        details = {}
2796        details["user"] = "git perforce import user"
2797        details["desc"] = ("Initial import of %s from the state at revision %s\n"
2798                           % (' '.join(self.depotPaths), revision))
2799        details["change"] = revision
2800        newestRevision = 0
2801
2802        fileCnt = 0
2803        fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
2804
2805        for info in p4CmdList(["files"] + fileArgs):
2806
2807            if 'code' in info and info['code'] == 'error':
2808                sys.stderr.write("p4 returned an error: %s\n"
2809                                 % info['data'])
2810                if info['data'].find("must refer to client") >= 0:
2811                    sys.stderr.write("This particular p4 error is misleading.\n")
2812                    sys.stderr.write("Perhaps the depot path was misspelled.\n");
2813                    sys.stderr.write("Depot path:  %s\n" % " ".join(self.depotPaths))
2814                sys.exit(1)
2815            if 'p4ExitCode' in info:
2816                sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
2817                sys.exit(1)
2818
2819
2820            change = int(info["change"])
2821            if change > newestRevision:
2822                newestRevision = change
2823
2824            if info["action"] in self.delete_actions:
2825                # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
2826                #fileCnt = fileCnt + 1
2827                continue
2828
2829            for prop in ["depotFile", "rev", "action", "type" ]:
2830                details["%s%s" % (prop, fileCnt)] = info[prop]
2831
2832            fileCnt = fileCnt + 1
2833
2834        details["change"] = newestRevision
2835
2836        # Use time from top-most change so that all git p4 clones of
2837        # the same p4 repo have the same commit SHA1s.
2838        res = p4_describe(newestRevision)
2839        details["time"] = res["time"]
2840
2841        self.updateOptionDict(details)
2842        try:
2843            self.commit(details, self.extractFilesFromCommit(details), self.branch)
2844        except IOError:
2845            print "IO error with git fast-import. Is your git version recent enough?"
2846            print self.gitError.read()
2847
2848
2849    def run(self, args):
2850        self.depotPaths = []
2851        self.changeRange = ""
2852        self.previousDepotPaths = []
2853        self.hasOrigin = False
2854
2855        # map from branch depot path to parent branch
2856        self.knownBranches = {}
2857        self.initialParents = {}
2858
2859        if self.importIntoRemotes:
2860            self.refPrefix = "refs/remotes/p4/"
2861        else:
2862            self.refPrefix = "refs/heads/p4/"
2863
2864        if self.syncWithOrigin:
2865            self.hasOrigin = originP4BranchesExist()
2866            if self.hasOrigin:
2867                if not self.silent:
2868                    print 'Syncing with origin first, using "git fetch origin"'
2869                system("git fetch origin")
2870
2871        branch_arg_given = bool(self.branch)
2872        if len(self.branch) == 0:
2873            self.branch = self.refPrefix + "master"
2874            if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
2875                system("git update-ref %s refs/heads/p4" % self.branch)
2876                system("git branch -D p4")
2877
2878        # accept either the command-line option, or the configuration variable
2879        if self.useClientSpec:
2880            # will use this after clone to set the variable
2881            self.useClientSpec_from_options = True
2882        else:
2883            if gitConfigBool("git-p4.useclientspec"):
2884                self.useClientSpec = True
2885        if self.useClientSpec:
2886            self.clientSpecDirs = getClientSpec()
2887
2888        # TODO: should always look at previous commits,
2889        # merge with previous imports, if possible.
2890        if args == []:
2891            if self.hasOrigin:
2892                createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
2893
2894            # branches holds mapping from branch name to sha1
2895            branches = p4BranchesInGit(self.importIntoRemotes)
2896
2897            # restrict to just this one, disabling detect-branches
2898            if branch_arg_given:
2899                short = self.branch.split("/")[-1]
2900                if short in branches:
2901                    self.p4BranchesInGit = [ short ]
2902            else:
2903                self.p4BranchesInGit = branches.keys()
2904
2905            if len(self.p4BranchesInGit) > 1:
2906                if not self.silent:
2907                    print "Importing from/into multiple branches"
2908                self.detectBranches = True
2909                for branch in branches.keys():
2910                    self.initialParents[self.refPrefix + branch] = \
2911                        branches[branch]
2912
2913            if self.verbose:
2914                print "branches: %s" % self.p4BranchesInGit
2915
2916            p4Change = 0
2917            for branch in self.p4BranchesInGit:
2918                logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
2919
2920                settings = extractSettingsGitLog(logMsg)
2921
2922                self.readOptions(settings)
2923                if (settings.has_key('depot-paths')
2924                    and settings.has_key ('change')):
2925                    change = int(settings['change']) + 1
2926                    p4Change = max(p4Change, change)
2927
2928                    depotPaths = sorted(settings['depot-paths'])
2929                    if self.previousDepotPaths == []:
2930                        self.previousDepotPaths = depotPaths
2931                    else:
2932                        paths = []
2933                        for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
2934                            prev_list = prev.split("/")
2935                            cur_list = cur.split("/")
2936                            for i in range(0, min(len(cur_list), len(prev_list))):
2937                                if cur_list[i] <> prev_list[i]:
2938                                    i = i - 1
2939                                    break
2940
2941                            paths.append ("/".join(cur_list[:i + 1]))
2942
2943                        self.previousDepotPaths = paths
2944
2945            if p4Change > 0:
2946                self.depotPaths = sorted(self.previousDepotPaths)
2947                self.changeRange = "@%s,#head" % p4Change
2948                if not self.silent and not self.detectBranches:
2949                    print "Performing incremental import into %s git branch" % self.branch
2950
2951        # accept multiple ref name abbreviations:
2952        #    refs/foo/bar/branch -> use it exactly
2953        #    p4/branch -> prepend refs/remotes/ or refs/heads/
2954        #    branch -> prepend refs/remotes/p4/ or refs/heads/p4/
2955        if not self.branch.startswith("refs/"):
2956            if self.importIntoRemotes:
2957                prepend = "refs/remotes/"
2958            else:
2959                prepend = "refs/heads/"
2960            if not self.branch.startswith("p4/"):
2961                prepend += "p4/"
2962            self.branch = prepend + self.branch
2963
2964        if len(args) == 0 and self.depotPaths:
2965            if not self.silent:
2966                print "Depot paths: %s" % ' '.join(self.depotPaths)
2967        else:
2968            if self.depotPaths and self.depotPaths != args:
2969                print ("previous import used depot path %s and now %s was specified. "
2970                       "This doesn't work!" % (' '.join (self.depotPaths),
2971                                               ' '.join (args)))
2972                sys.exit(1)
2973
2974            self.depotPaths = sorted(args)
2975
2976        revision = ""
2977        self.users = {}
2978
2979        # Make sure no revision specifiers are used when --changesfile
2980        # is specified.
2981        bad_changesfile = False
2982        if len(self.changesFile) > 0:
2983            for p in self.depotPaths:
2984                if p.find("@") >= 0 or p.find("#") >= 0:
2985                    bad_changesfile = True
2986                    break
2987        if bad_changesfile:
2988            die("Option --changesfile is incompatible with revision specifiers")
2989
2990        newPaths = []
2991        for p in self.depotPaths:
2992            if p.find("@") != -1:
2993                atIdx = p.index("@")
2994                self.changeRange = p[atIdx:]
2995                if self.changeRange == "@all":
2996                    self.changeRange = ""
2997                elif ',' not in self.changeRange:
2998                    revision = self.changeRange
2999                    self.changeRange = ""
3000                p = p[:atIdx]
3001            elif p.find("#") != -1:
3002                hashIdx = p.index("#")
3003                revision = p[hashIdx:]
3004                p = p[:hashIdx]
3005            elif self.previousDepotPaths == []:
3006                # pay attention to changesfile, if given, else import
3007                # the entire p4 tree at the head revision
3008                if len(self.changesFile) == 0:
3009                    revision = "#head"
3010
3011            p = re.sub ("\.\.\.$", "", p)
3012            if not p.endswith("/"):
3013                p += "/"
3014
3015            newPaths.append(p)
3016
3017        self.depotPaths = newPaths
3018
3019        # --detect-branches may change this for each branch
3020        self.branchPrefixes = self.depotPaths
3021
3022        self.loadUserMapFromCache()
3023        self.labels = {}
3024        if self.detectLabels:
3025            self.getLabels();
3026
3027        if self.detectBranches:
3028            ## FIXME - what's a P4 projectName ?
3029            self.projectName = self.guessProjectName()
3030
3031            if self.hasOrigin:
3032                self.getBranchMappingFromGitBranches()
3033            else:
3034                self.getBranchMapping()
3035            if self.verbose:
3036                print "p4-git branches: %s" % self.p4BranchesInGit
3037                print "initial parents: %s" % self.initialParents
3038            for b in self.p4BranchesInGit:
3039                if b != "master":
3040
3041                    ## FIXME
3042                    b = b[len(self.projectName):]
3043                self.createdBranches.add(b)
3044
3045        self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
3046
3047        self.importProcess = subprocess.Popen(["git", "fast-import"],
3048                                              stdin=subprocess.PIPE,
3049                                              stdout=subprocess.PIPE,
3050                                              stderr=subprocess.PIPE);
3051        self.gitOutput = self.importProcess.stdout
3052        self.gitStream = self.importProcess.stdin
3053        self.gitError = self.importProcess.stderr
3054
3055        if revision:
3056            self.importHeadRevision(revision)
3057        else:
3058            changes = []
3059
3060            if len(self.changesFile) > 0:
3061                output = open(self.changesFile).readlines()
3062                changeSet = set()
3063                for line in output:
3064                    changeSet.add(int(line))
3065
3066                for change in changeSet:
3067                    changes.append(change)
3068
3069                changes.sort()
3070            else:
3071                # catch "git p4 sync" with no new branches, in a repo that
3072                # does not have any existing p4 branches
3073                if len(args) == 0:
3074                    if not self.p4BranchesInGit:
3075                        die("No remote p4 branches.  Perhaps you never did \"git p4 clone\" in here.")
3076
3077                    # The default branch is master, unless --branch is used to
3078                    # specify something else.  Make sure it exists, or complain
3079                    # nicely about how to use --branch.
3080                    if not self.detectBranches:
3081                        if not branch_exists(self.branch):
3082                            if branch_arg_given:
3083                                die("Error: branch %s does not exist." % self.branch)
3084                            else:
3085                                die("Error: no branch %s; perhaps specify one with --branch." %
3086                                    self.branch)
3087
3088                if self.verbose:
3089                    print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
3090                                                              self.changeRange)
3091                changes = p4ChangesForPaths(self.depotPaths, self.changeRange, self.changes_block_size)
3092
3093                if len(self.maxChanges) > 0:
3094                    changes = changes[:min(int(self.maxChanges), len(changes))]
3095
3096            if len(changes) == 0:
3097                if not self.silent:
3098                    print "No changes to import!"
3099            else:
3100                if not self.silent and not self.detectBranches:
3101                    print "Import destination: %s" % self.branch
3102
3103                self.updatedBranches = set()
3104
3105                if not self.detectBranches:
3106                    if args:
3107                        # start a new branch
3108                        self.initialParent = ""
3109                    else:
3110                        # build on a previous revision
3111                        self.initialParent = parseRevision(self.branch)
3112
3113                self.importChanges(changes)
3114
3115                if not self.silent:
3116                    print ""
3117                    if len(self.updatedBranches) > 0:
3118                        sys.stdout.write("Updated branches: ")
3119                        for b in self.updatedBranches:
3120                            sys.stdout.write("%s " % b)
3121                        sys.stdout.write("\n")
3122
3123        if gitConfigBool("git-p4.importLabels"):
3124            self.importLabels = True
3125
3126        if self.importLabels:
3127            p4Labels = getP4Labels(self.depotPaths)
3128            gitTags = getGitTags()
3129
3130            missingP4Labels = p4Labels - gitTags
3131            self.importP4Labels(self.gitStream, missingP4Labels)
3132
3133        self.gitStream.close()
3134        if self.importProcess.wait() != 0:
3135            die("fast-import failed: %s" % self.gitError.read())
3136        self.gitOutput.close()
3137        self.gitError.close()
3138
3139        # Cleanup temporary branches created during import
3140        if self.tempBranches != []:
3141            for branch in self.tempBranches:
3142                read_pipe("git update-ref -d %s" % branch)
3143            os.rmdir(os.path.join(os.environ.get("GIT_DIR", ".git"), self.tempBranchLocation))
3144
3145        # Create a symbolic ref p4/HEAD pointing to p4/<branch> to allow
3146        # a convenient shortcut refname "p4".
3147        if self.importIntoRemotes:
3148            head_ref = self.refPrefix + "HEAD"
3149            if not gitBranchExists(head_ref) and gitBranchExists(self.branch):
3150                system(["git", "symbolic-ref", head_ref, self.branch])
3151
3152        return True
3153
3154class P4Rebase(Command):
3155    def __init__(self):
3156        Command.__init__(self)
3157        self.options = [
3158                optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
3159        ]
3160        self.importLabels = False
3161        self.description = ("Fetches the latest revision from perforce and "
3162                            + "rebases the current work (branch) against it")
3163
3164    def run(self, args):
3165        sync = P4Sync()
3166        sync.importLabels = self.importLabels
3167        sync.run([])
3168
3169        return self.rebase()
3170
3171    def rebase(self):
3172        if os.system("git update-index --refresh") != 0:
3173            die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
3174        if len(read_pipe("git diff-index HEAD --")) > 0:
3175            die("You have uncommitted changes. Please commit them before rebasing or stash them away with git stash.");
3176
3177        [upstream, settings] = findUpstreamBranchPoint()
3178        if len(upstream) == 0:
3179            die("Cannot find upstream branchpoint for rebase")
3180
3181        # the branchpoint may be p4/foo~3, so strip off the parent
3182        upstream = re.sub("~[0-9]+$", "", upstream)
3183
3184        print "Rebasing the current branch onto %s" % upstream
3185        oldHead = read_pipe("git rev-parse HEAD").strip()
3186        system("git rebase %s" % upstream)
3187        system("git diff-tree --stat --summary -M %s HEAD --" % oldHead)
3188        return True
3189
3190class P4Clone(P4Sync):
3191    def __init__(self):
3192        P4Sync.__init__(self)
3193        self.description = "Creates a new git repository and imports from Perforce into it"
3194        self.usage = "usage: %prog [options] //depot/path[@revRange]"
3195        self.options += [
3196            optparse.make_option("--destination", dest="cloneDestination",
3197                                 action='store', default=None,
3198                                 help="where to leave result of the clone"),
3199            optparse.make_option("--bare", dest="cloneBare",
3200                                 action="store_true", default=False),
3201        ]
3202        self.cloneDestination = None
3203        self.needsGit = False
3204        self.cloneBare = False
3205
3206    def defaultDestination(self, args):
3207        ## TODO: use common prefix of args?
3208        depotPath = args[0]
3209        depotDir = re.sub("(@[^@]*)$", "", depotPath)
3210        depotDir = re.sub("(#[^#]*)$", "", depotDir)
3211        depotDir = re.sub(r"\.\.\.$", "", depotDir)
3212        depotDir = re.sub(r"/$", "", depotDir)
3213        return os.path.split(depotDir)[1]
3214
3215    def run(self, args):
3216        if len(args) < 1:
3217            return False
3218
3219        if self.keepRepoPath and not self.cloneDestination:
3220            sys.stderr.write("Must specify destination for --keep-path\n")
3221            sys.exit(1)
3222
3223        depotPaths = args
3224
3225        if not self.cloneDestination and len(depotPaths) > 1:
3226            self.cloneDestination = depotPaths[-1]
3227            depotPaths = depotPaths[:-1]
3228
3229        self.cloneExclude = ["/"+p for p in self.cloneExclude]
3230        for p in depotPaths:
3231            if not p.startswith("//"):
3232                sys.stderr.write('Depot paths must start with "//": %s\n' % p)
3233                return False
3234
3235        if not self.cloneDestination:
3236            self.cloneDestination = self.defaultDestination(args)
3237
3238        print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
3239
3240        if not os.path.exists(self.cloneDestination):
3241            os.makedirs(self.cloneDestination)
3242        chdir(self.cloneDestination)
3243
3244        init_cmd = [ "git", "init" ]
3245        if self.cloneBare:
3246            init_cmd.append("--bare")
3247        retcode = subprocess.call(init_cmd)
3248        if retcode:
3249            raise CalledProcessError(retcode, init_cmd)
3250
3251        if not P4Sync.run(self, depotPaths):
3252            return False
3253
3254        # create a master branch and check out a work tree
3255        if gitBranchExists(self.branch):
3256            system([ "git", "branch", "master", self.branch ])
3257            if not self.cloneBare:
3258                system([ "git", "checkout", "-f" ])
3259        else:
3260            print 'Not checking out any branch, use ' \
3261                  '"git checkout -q -b master <branch>"'
3262
3263        # auto-set this variable if invoked with --use-client-spec
3264        if self.useClientSpec_from_options:
3265            system("git config --bool git-p4.useclientspec true")
3266
3267        return True
3268
3269class P4Branches(Command):
3270    def __init__(self):
3271        Command.__init__(self)
3272        self.options = [ ]
3273        self.description = ("Shows the git branches that hold imports and their "
3274                            + "corresponding perforce depot paths")
3275        self.verbose = False
3276
3277    def run(self, args):
3278        if originP4BranchesExist():
3279            createOrUpdateBranchesFromOrigin()
3280
3281        cmdline = "git rev-parse --symbolic "
3282        cmdline += " --remotes"
3283
3284        for line in read_pipe_lines(cmdline):
3285            line = line.strip()
3286
3287            if not line.startswith('p4/') or line == "p4/HEAD":
3288                continue
3289            branch = line
3290
3291            log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
3292            settings = extractSettingsGitLog(log)
3293
3294            print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
3295        return True
3296
3297class HelpFormatter(optparse.IndentedHelpFormatter):
3298    def __init__(self):
3299        optparse.IndentedHelpFormatter.__init__(self)
3300
3301    def format_description(self, description):
3302        if description:
3303            return description + "\n"
3304        else:
3305            return ""
3306
3307def printUsage(commands):
3308    print "usage: %s <command> [options]" % sys.argv[0]
3309    print ""
3310    print "valid commands: %s" % ", ".join(commands)
3311    print ""
3312    print "Try %s <command> --help for command specific help." % sys.argv[0]
3313    print ""
3314
3315commands = {
3316    "debug" : P4Debug,
3317    "submit" : P4Submit,
3318    "commit" : P4Submit,
3319    "sync" : P4Sync,
3320    "rebase" : P4Rebase,
3321    "clone" : P4Clone,
3322    "rollback" : P4RollBack,
3323    "branches" : P4Branches
3324}
3325
3326
3327def main():
3328    if len(sys.argv[1:]) == 0:
3329        printUsage(commands.keys())
3330        sys.exit(2)
3331
3332    cmdName = sys.argv[1]
3333    try:
3334        klass = commands[cmdName]
3335        cmd = klass()
3336    except KeyError:
3337        print "unknown command %s" % cmdName
3338        print ""
3339        printUsage(commands.keys())
3340        sys.exit(2)
3341
3342    options = cmd.options
3343    cmd.gitdir = os.environ.get("GIT_DIR", None)
3344
3345    args = sys.argv[2:]
3346
3347    options.append(optparse.make_option("--verbose", "-v", dest="verbose", action="store_true"))
3348    if cmd.needsGit:
3349        options.append(optparse.make_option("--git-dir", dest="gitdir"))
3350
3351    parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
3352                                   options,
3353                                   description = cmd.description,
3354                                   formatter = HelpFormatter())
3355
3356    (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
3357    global verbose
3358    verbose = cmd.verbose
3359    if cmd.needsGit:
3360        if cmd.gitdir == None:
3361            cmd.gitdir = os.path.abspath(".git")
3362            if not isValidGitDir(cmd.gitdir):
3363                cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
3364                if os.path.exists(cmd.gitdir):
3365                    cdup = read_pipe("git rev-parse --show-cdup").strip()
3366                    if len(cdup) > 0:
3367                        chdir(cdup);
3368
3369        if not isValidGitDir(cmd.gitdir):
3370            if isValidGitDir(cmd.gitdir + "/.git"):
3371                cmd.gitdir += "/.git"
3372            else:
3373                die("fatal: cannot locate git repository at %s" % cmd.gitdir)
3374
3375        os.environ["GIT_DIR"] = cmd.gitdir
3376
3377    if not cmd.run(args):
3378        parser.print_help()
3379        sys.exit(2)
3380
3381
3382if __name__ == '__main__':
3383    main()